diff --git a/.buildkite/pipelines/dra-workflow.yml b/.buildkite/pipelines/dra-workflow.yml index e7bf19816356f..32a2b7d22134a 100644 --- a/.buildkite/pipelines/dra-workflow.yml +++ b/.buildkite/pipelines/dra-workflow.yml @@ -7,6 +7,7 @@ steps: image: family/elasticsearch-ubuntu-2204 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait # The hadoop build depends on the ES artifact # So let's trigger the hadoop build any time we build a new staging artifact diff --git a/.buildkite/pipelines/intake.template.yml b/.buildkite/pipelines/intake.template.yml index f530f237113a9..1a513971b2c10 100644 --- a/.buildkite/pipelines/intake.template.yml +++ b/.buildkite/pipelines/intake.template.yml @@ -7,6 +7,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -16,6 +17,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -24,6 +26,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -32,6 +35,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -40,6 +44,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -48,6 +53,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -61,6 +67,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -71,6 +78,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml index c5b079c39fbc1..4124d4e550d11 100644 --- a/.buildkite/pipelines/intake.yml +++ b/.buildkite/pipelines/intake.yml @@ -8,6 +8,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -17,6 +18,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -25,6 +27,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -33,6 +36,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -41,6 +45,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -49,6 +54,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -56,12 +62,13 @@ steps: timeout_in_minutes: 300 matrix: setup: - BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"] + BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -72,6 +79,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - trigger: elasticsearch-dra-workflow label: Trigger DRA snapshot workflow diff --git a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml index 8cf2a8aacbece..1f69b8faa7ab4 100644 --- a/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml +++ b/.buildkite/pipelines/lucene-snapshot/build-snapshot.yml @@ -15,6 +15,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait - trigger: "elasticsearch-lucene-snapshot-tests" build: diff --git a/.buildkite/pipelines/lucene-snapshot/run-tests.yml b/.buildkite/pipelines/lucene-snapshot/run-tests.yml index c76c54a56494e..49c3396488d82 100644 --- a/.buildkite/pipelines/lucene-snapshot/run-tests.yml +++ b/.buildkite/pipelines/lucene-snapshot/run-tests.yml @@ -7,6 +7,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - wait: null - label: part1 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart1 @@ -16,6 +17,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part2 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart2 timeout_in_minutes: 300 @@ -24,6 +26,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part3 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart3 timeout_in_minutes: 300 @@ -32,6 +35,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part4 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart4 timeout_in_minutes: 300 @@ -40,6 +44,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: part5 command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-task-input-files checkPart5 timeout_in_minutes: 300 @@ -48,6 +53,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: bwc-snapshots steps: - label: "{{matrix.BWC_VERSION}} / bwc-snapshots" @@ -64,6 +70,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: "{{matrix.BWC_VERSION}}" - label: rest-compat @@ -74,3 +81,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/periodic-packaging.bwc.template.yml b/.buildkite/pipelines/periodic-packaging.bwc.template.yml index b06bc80d3535d..8a6fa2553b204 100644 --- a/.buildkite/pipelines/periodic-packaging.bwc.template.yml +++ b/.buildkite/pipelines/periodic-packaging.bwc.template.yml @@ -11,5 +11,6 @@ image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml index 378a7c5c9c5d2..4217fc91bf0fd 100644 --- a/.buildkite/pipelines/periodic-packaging.yml +++ b/.buildkite/pipelines/periodic-packaging.yml @@ -46,6 +46,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.0.1 @@ -62,6 +63,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.1.1 @@ -78,6 +80,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.2.1 @@ -94,6 +97,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.3.2 @@ -110,6 +114,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.4.2 @@ -126,6 +131,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.5.2 @@ -142,6 +148,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.6.2 @@ -158,6 +165,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.7.1 @@ -174,6 +182,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.8.1 @@ -190,6 +199,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.9.3 @@ -206,6 +216,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.10.2 @@ -222,6 +233,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.11.2 @@ -238,6 +250,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.12.1 @@ -254,6 +267,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.13.4 @@ -270,6 +284,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.14.2 @@ -286,6 +301,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.15.2 @@ -302,11 +318,12 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 7.16.3 - - label: "{{matrix.image}} / 7.17.22 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.22 + - label: "{{matrix.image}} / 7.17.23 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.23 timeout_in_minutes: 300 matrix: setup: @@ -318,8 +335,9 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: - BWC_VERSION: 7.17.22 + BWC_VERSION: 7.17.23 - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.1 @@ -334,6 +352,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.0.1 @@ -350,6 +369,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.1.3 @@ -366,6 +386,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.2.3 @@ -382,6 +403,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.3.3 @@ -398,6 +420,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.4.3 @@ -414,6 +437,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.5.3 @@ -430,6 +454,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.6.2 @@ -446,6 +471,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.7.1 @@ -462,6 +488,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.8.2 @@ -478,6 +505,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.9.2 @@ -494,6 +522,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.10.4 @@ -510,6 +539,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.11.4 @@ -526,6 +556,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.12.2 @@ -542,11 +573,12 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.13.4 - - label: "{{matrix.image}} / 8.14.1 / packaging-tests-upgrade" - command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.1 + - label: "{{matrix.image}} / 8.14.2 / packaging-tests-upgrade" + command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.14.2 timeout_in_minutes: 300 matrix: setup: @@ -558,8 +590,9 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: - BWC_VERSION: 8.14.1 + BWC_VERSION: 8.14.2 - label: "{{matrix.image}} / 8.15.0 / packaging-tests-upgrade" command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.0 @@ -574,6 +607,7 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: 8.15.0 diff --git a/.buildkite/pipelines/periodic-platform-support.yml b/.buildkite/pipelines/periodic-platform-support.yml index d8c5d55fc7e4f..867ebe41ed6af 100644 --- a/.buildkite/pipelines/periodic-platform-support.yml +++ b/.buildkite/pipelines/periodic-platform-support.yml @@ -30,6 +30,7 @@ steps: localSsds: 1 localSsdInterface: nvme machineType: custom-32-98304 + diskSizeGb: 250 env: {} - group: platform-support-windows steps: diff --git a/.buildkite/pipelines/periodic.bwc.template.yml b/.buildkite/pipelines/periodic.bwc.template.yml index 43a0a7438d656..b22270dbf221c 100644 --- a/.buildkite/pipelines/periodic.bwc.template.yml +++ b/.buildkite/pipelines/periodic.bwc.template.yml @@ -7,6 +7,7 @@ machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION retry: diff --git a/.buildkite/pipelines/periodic.template.yml b/.buildkite/pipelines/periodic.template.yml index 207a332ed6717..87e30a0ea73ba 100644 --- a/.buildkite/pipelines/periodic.template.yml +++ b/.buildkite/pipelines/periodic.template.yml @@ -25,6 +25,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: example-plugins command: |- cd $$WORKSPACE/plugins/examples @@ -36,6 +37,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: java-fips-matrix steps: - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" @@ -57,6 +59,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -73,6 +76,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -101,6 +105,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -121,6 +126,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -156,6 +162,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / azure command: | export azure_storage_container=elasticsearch-ci-thirdparty @@ -170,6 +177,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / gcs command: | export google_storage_bucket=elasticsearch-ci-thirdparty @@ -184,6 +192,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / geoip command: | .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" @@ -193,6 +202,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 @@ -207,6 +217,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: @@ -217,6 +228,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 if: build.branch == "main" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency @@ -225,6 +237,7 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-2 + diskSizeGb: 250 - label: check-branch-protection-rules command: .buildkite/scripts/branch-protection.sh timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml index 1726f0f29fa92..06e7ffbc8fb1c 100644 --- a/.buildkite/pipelines/periodic.yml +++ b/.buildkite/pipelines/periodic.yml @@ -11,6 +11,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.0.1 retry: @@ -30,6 +31,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.1.1 retry: @@ -49,6 +51,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.2.1 retry: @@ -68,6 +71,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.3.2 retry: @@ -87,6 +91,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.4.2 retry: @@ -106,6 +111,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.5.2 retry: @@ -125,6 +131,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.6.2 retry: @@ -144,6 +151,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.7.1 retry: @@ -163,6 +171,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.8.1 retry: @@ -182,6 +191,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.9.3 retry: @@ -201,6 +211,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.10.2 retry: @@ -220,6 +231,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.11.2 retry: @@ -239,6 +251,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.12.1 retry: @@ -258,6 +271,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.13.4 retry: @@ -277,6 +291,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.14.2 retry: @@ -296,6 +311,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.15.2 retry: @@ -315,6 +331,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 7.16.3 retry: @@ -325,8 +342,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 7.17.22 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.22#bwcTest + - label: 7.17.23 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.23#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -334,8 +351,9 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: - BWC_VERSION: 7.17.22 + BWC_VERSION: 7.17.23 retry: automatic: - exit_status: "-1" @@ -353,6 +371,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.0.1 retry: @@ -372,6 +391,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.1.3 retry: @@ -391,6 +411,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.2.3 retry: @@ -410,6 +431,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.3.3 retry: @@ -429,6 +451,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.4.3 retry: @@ -448,6 +471,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.5.3 retry: @@ -467,6 +491,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.6.2 retry: @@ -486,6 +511,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.7.1 retry: @@ -505,6 +531,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.8.2 retry: @@ -524,6 +551,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.9.2 retry: @@ -543,6 +571,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.10.4 retry: @@ -562,6 +591,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.11.4 retry: @@ -581,6 +611,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.12.2 retry: @@ -600,6 +631,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.13.4 retry: @@ -610,8 +642,8 @@ steps: - signal_reason: agent_stop limit: 3 - - label: 8.14.1 / bwc - command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.1#bwcTest + - label: 8.14.2 / bwc + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.14.2#bwcTest timeout_in_minutes: 300 agents: provider: gcp @@ -619,8 +651,9 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: - BWC_VERSION: 8.14.1 + BWC_VERSION: 8.14.2 retry: automatic: - exit_status: "-1" @@ -638,6 +671,7 @@ steps: machineType: n1-standard-32 buildDirectory: /dev/shm/bk preemptible: true + diskSizeGb: 250 env: BWC_VERSION: 8.15.0 retry: @@ -672,6 +706,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: example-plugins command: |- cd $$WORKSPACE/plugins/examples @@ -683,6 +718,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - group: java-fips-matrix steps: - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" @@ -704,6 +740,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -714,12 +751,13 @@ steps: setup: ES_RUNTIME_JAVA: - openjdk17 - BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"] + BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -748,6 +786,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" GRADLE_TASK: "{{matrix.GRADLE_TASK}}" @@ -762,12 +801,13 @@ steps: - openjdk21 - openjdk22 - openjdk23 - BWC_VERSION: ["7.17.22", "8.14.1", "8.15.0"] + BWC_VERSION: ["7.17.23", "8.14.2", "8.15.0"] agents: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" BWC_VERSION: "{{matrix.BWC_VERSION}}" @@ -803,6 +843,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / azure command: | export azure_storage_container=elasticsearch-ci-thirdparty @@ -817,6 +858,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / gcs command: | export google_storage_bucket=elasticsearch-ci-thirdparty @@ -831,6 +873,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / geoip command: | .ci/scripts/run-gradle.sh :modules:ingest-geoip:internalClusterTest -Dtests.jvm.argline="-Dgeoip_use_service=true" @@ -840,6 +883,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: third-party / s3 command: | export amazon_s3_bucket=elasticsearch-ci.us-west-2 @@ -854,6 +898,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 - label: Upload Snyk Dependency Graph command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH env: @@ -864,6 +909,7 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-8 buildDirectory: /dev/shm/bk + diskSizeGb: 250 if: build.branch == "main" || build.branch == "7.17" - label: check-branch-consistency command: .ci/scripts/run-gradle.sh branchConsistency @@ -872,6 +918,7 @@ steps: provider: gcp image: family/elasticsearch-ubuntu-2004 machineType: n2-standard-2 + diskSizeGb: 250 - label: check-branch-protection-rules command: .buildkite/scripts/branch-protection.sh timeout_in_minutes: 5 diff --git a/.buildkite/pipelines/pull-request/build-benchmark.yml b/.buildkite/pipelines/pull-request/build-benchmark.yml index 8d3215b8393ce..96330bee03638 100644 --- a/.buildkite/pipelines/pull-request/build-benchmark.yml +++ b/.buildkite/pipelines/pull-request/build-benchmark.yml @@ -22,3 +22,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/bwc-snapshots.yml b/.buildkite/pipelines/pull-request/bwc-snapshots.yml index 5a9fc2d938ac0..8f59e593b286f 100644 --- a/.buildkite/pipelines/pull-request/bwc-snapshots.yml +++ b/.buildkite/pipelines/pull-request/bwc-snapshots.yml @@ -18,3 +18,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: n1-standard-32 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/cloud-deploy.yml b/.buildkite/pipelines/pull-request/cloud-deploy.yml index ce8e8206d51ff..2932f874c5cf8 100644 --- a/.buildkite/pipelines/pull-request/cloud-deploy.yml +++ b/.buildkite/pipelines/pull-request/cloud-deploy.yml @@ -11,3 +11,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/docs-check.yml b/.buildkite/pipelines/pull-request/docs-check.yml index 2201eb2d1e4ea..3bf1e43697a7c 100644 --- a/.buildkite/pipelines/pull-request/docs-check.yml +++ b/.buildkite/pipelines/pull-request/docs-check.yml @@ -12,3 +12,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/eql-correctness.yml b/.buildkite/pipelines/pull-request/eql-correctness.yml index 8f7ca6942c0e9..d85827d10e886 100644 --- a/.buildkite/pipelines/pull-request/eql-correctness.yml +++ b/.buildkite/pipelines/pull-request/eql-correctness.yml @@ -7,3 +7,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/example-plugins.yml b/.buildkite/pipelines/pull-request/example-plugins.yml index 18d0de6594980..fb4a17fb214cb 100644 --- a/.buildkite/pipelines/pull-request/example-plugins.yml +++ b/.buildkite/pipelines/pull-request/example-plugins.yml @@ -16,3 +16,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/full-bwc.yml b/.buildkite/pipelines/pull-request/full-bwc.yml index d3fa8eccaf7d9..c404069bd0e60 100644 --- a/.buildkite/pipelines/pull-request/full-bwc.yml +++ b/.buildkite/pipelines/pull-request/full-bwc.yml @@ -13,3 +13,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml b/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml index c62cf23310422..970dafbb28647 100644 --- a/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml +++ b/.buildkite/pipelines/pull-request/packaging-upgrade-tests.yml @@ -18,5 +18,6 @@ steps: image: family/elasticsearch-{{matrix.image}} machineType: custom-16-32768 buildDirectory: /dev/shm/bk + diskSizeGb: 250 env: BWC_VERSION: $BWC_VERSION diff --git a/.buildkite/pipelines/pull-request/part-1-fips.yml b/.buildkite/pipelines/pull-request/part-1-fips.yml index 42f930c1bde9a..99544e7f5a80b 100644 --- a/.buildkite/pipelines/pull-request/part-1-fips.yml +++ b/.buildkite/pipelines/pull-request/part-1-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-1.yml b/.buildkite/pipelines/pull-request/part-1.yml index 3d467c6c41e43..b4b9d5469ec41 100644 --- a/.buildkite/pipelines/pull-request/part-1.yml +++ b/.buildkite/pipelines/pull-request/part-1.yml @@ -7,3 +7,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-2-fips.yml b/.buildkite/pipelines/pull-request/part-2-fips.yml index 6a3647ceb50ae..36a9801547d78 100644 --- a/.buildkite/pipelines/pull-request/part-2-fips.yml +++ b/.buildkite/pipelines/pull-request/part-2-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-2.yml b/.buildkite/pipelines/pull-request/part-2.yml index 43de69bbcd945..12bd78cf895fd 100644 --- a/.buildkite/pipelines/pull-request/part-2.yml +++ b/.buildkite/pipelines/pull-request/part-2.yml @@ -7,3 +7,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-3-fips.yml b/.buildkite/pipelines/pull-request/part-3-fips.yml index cee3ea153acb9..4a2df3026e782 100644 --- a/.buildkite/pipelines/pull-request/part-3-fips.yml +++ b/.buildkite/pipelines/pull-request/part-3-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-3.yml b/.buildkite/pipelines/pull-request/part-3.yml index 12abae7634822..6991c05da85c6 100644 --- a/.buildkite/pipelines/pull-request/part-3.yml +++ b/.buildkite/pipelines/pull-request/part-3.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-4-fips.yml b/.buildkite/pipelines/pull-request/part-4-fips.yml index 11a50456ca4c0..734f8af816895 100644 --- a/.buildkite/pipelines/pull-request/part-4-fips.yml +++ b/.buildkite/pipelines/pull-request/part-4-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-4.yml b/.buildkite/pipelines/pull-request/part-4.yml index af11f08953d07..59f2f2898a590 100644 --- a/.buildkite/pipelines/pull-request/part-4.yml +++ b/.buildkite/pipelines/pull-request/part-4.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-5-fips.yml b/.buildkite/pipelines/pull-request/part-5-fips.yml index 4e193ac751086..801b812bb99c0 100644 --- a/.buildkite/pipelines/pull-request/part-5-fips.yml +++ b/.buildkite/pipelines/pull-request/part-5-fips.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/part-5.yml b/.buildkite/pipelines/pull-request/part-5.yml index 306ce7533d0ed..c7e50631d1cdd 100644 --- a/.buildkite/pipelines/pull-request/part-5.yml +++ b/.buildkite/pipelines/pull-request/part-5.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/precommit.yml b/.buildkite/pipelines/pull-request/precommit.yml index f6548dfeed9b2..8d1458b1b60c8 100644 --- a/.buildkite/pipelines/pull-request/precommit.yml +++ b/.buildkite/pipelines/pull-request/precommit.yml @@ -10,3 +10,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/rest-compatibility.yml b/.buildkite/pipelines/pull-request/rest-compatibility.yml index a69810e23d960..16144a2a0780f 100644 --- a/.buildkite/pipelines/pull-request/rest-compatibility.yml +++ b/.buildkite/pipelines/pull-request/rest-compatibility.yml @@ -9,3 +9,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.buildkite/pipelines/pull-request/validate-changelogs.yml b/.buildkite/pipelines/pull-request/validate-changelogs.yml index 9451d321a9b39..296ef11637118 100644 --- a/.buildkite/pipelines/pull-request/validate-changelogs.yml +++ b/.buildkite/pipelines/pull-request/validate-changelogs.yml @@ -7,3 +7,4 @@ steps: image: family/elasticsearch-ubuntu-2004 machineType: custom-32-98304 buildDirectory: /dev/shm/bk + diskSizeGb: 250 diff --git a/.ci/bwcVersions b/.ci/bwcVersions index 3aa17cc370296..bce556e9fc352 100644 --- a/.ci/bwcVersions +++ b/.ci/bwcVersions @@ -16,7 +16,7 @@ BWC_VERSION: - "7.14.2" - "7.15.2" - "7.16.3" - - "7.17.22" + - "7.17.23" - "8.0.1" - "8.1.3" - "8.2.3" @@ -31,5 +31,5 @@ BWC_VERSION: - "8.11.4" - "8.12.2" - "8.13.4" - - "8.14.1" + - "8.14.2" - "8.15.0" diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions index f802829f6ec8a..5fc4b6c072899 100644 --- a/.ci/snapshotBwcVersions +++ b/.ci/snapshotBwcVersions @@ -1,4 +1,4 @@ BWC_VERSION: - - "7.17.22" - - "8.14.1" + - "7.17.23" + - "8.14.2" - "8.15.0" diff --git a/.gitattributes b/.gitattributes index 6a8de5462ec3f..04881c92ede00 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,6 +4,8 @@ CHANGELOG.asciidoc merge=union # Windows build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/*.asciidoc text eol=lf +x-pack/plugin/esql/compute/src/main/generated/** linguist-generated=true +x-pack/plugin/esql/compute/src/main/generated-src/** linguist-generated=true x-pack/plugin/esql/src/main/antlr/*.tokens linguist-generated=true x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*.interp linguist-generated=true x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java linguist-generated=true diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e1b476b657267..0f7e3073ed022 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -20,6 +20,9 @@ x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/Monito x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet @elastic/fleet x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet +# Logstash +libs/logstash-bridge @elastic/logstash + # Kibana Security x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle index 29b8ddde5fb2b..8753d4a4762b7 100644 --- a/benchmarks/build.gradle +++ b/benchmarks/build.gradle @@ -41,7 +41,7 @@ dependencies { api(project(':x-pack:plugin:esql-core')) api(project(':x-pack:plugin:esql')) api(project(':x-pack:plugin:esql:compute')) - implementation project(path: ':libs:elasticsearch-vec') + implementation project(path: ':libs:elasticsearch-simdvec') expression(project(path: ':modules:lang-expression', configuration: 'zip')) painless(project(path: ':modules:lang-painless', configuration: 'zip')) api "org.openjdk.jmh:jmh-core:$versions.jmh" diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java index fe6ba4da29f3b..0a4c836e2a6cf 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java @@ -56,7 +56,7 @@ public class DistanceFunctionBenchmark { @Param({ "96" }) private int dims; - @Param({ "dot", "cosine", "l1", "l2" }) + @Param({ "dot", "cosine", "l1", "l2", "hamming" }) private String function; @Param({ "knn", "binary" }) @@ -330,6 +330,18 @@ public void execute(Consumer consumer) { } } + private static class HammingKnnByteBenchmarkFunction extends KnnByteBenchmarkFunction { + + private HammingKnnByteBenchmarkFunction(int dims) { + super(dims); + } + + @Override + public void execute(Consumer consumer) { + new ByteKnnDenseVector(docVector).hamming(queryVector); + } + } + private static class L1BinaryFloatBenchmarkFunction extends BinaryFloatBenchmarkFunction { private L1BinaryFloatBenchmarkFunction(int dims) { @@ -354,6 +366,18 @@ public void execute(Consumer consumer) { } } + private static class HammingBinaryByteBenchmarkFunction extends BinaryByteBenchmarkFunction { + + private HammingBinaryByteBenchmarkFunction(int dims) { + super(dims); + } + + @Override + public void execute(Consumer consumer) { + new ByteBinaryDenseVector(vectorValue, docVector, dims).hamming(queryVector); + } + } + private static class L2KnnFloatBenchmarkFunction extends KnnFloatBenchmarkFunction { private L2KnnFloatBenchmarkFunction(int dims) { @@ -454,6 +478,11 @@ public void setBenchmarkFunction() { case "binary" -> new L2BinaryByteBenchmarkFunction(dims); default -> throw new UnsupportedOperationException("unexpected type [" + type + "]"); }; + case "hamming" -> benchmarkFunction = switch (type) { + case "knn" -> new HammingKnnByteBenchmarkFunction(dims); + case "binary" -> new HammingBinaryByteBenchmarkFunction(dims); + default -> throw new UnsupportedOperationException("unexpected type [" + type + "]"); + }; default -> throw new UnsupportedOperationException("unexpected function [" + function + "]"); } } diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java index 68e0f2151d1d7..89b512920cb09 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java @@ -22,7 +22,7 @@ import org.apache.lucene.util.quantization.ScalarQuantizer; import org.elasticsearch.common.logging.LogConfigurator; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.vec.VectorScorerFactory; +import org.elasticsearch.simdvec.VectorScorerFactory; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -41,8 +41,8 @@ import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.vec.VectorSimilarityType.DOT_PRODUCT; -import static org.elasticsearch.vec.VectorSimilarityType.EUCLIDEAN; +import static org.elasticsearch.simdvec.VectorSimilarityType.DOT_PRODUCT; +import static org.elasticsearch.simdvec.VectorSimilarityType.EUCLIDEAN; @Fork(value = 1, jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" }) @Warmup(iterations = 3, time = 3) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java index da8cd783d0365..13f265388fe3f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionModuleCheckTaskProvider.java @@ -62,8 +62,8 @@ public class InternalDistributionModuleCheckTaskProvider { "org.elasticsearch.preallocate", "org.elasticsearch.securesm", "org.elasticsearch.server", + "org.elasticsearch.simdvec", "org.elasticsearch.tdigest", - "org.elasticsearch.vec", "org.elasticsearch.xcontent" ); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java index 58b967d0a7722..4263ef2b1f76f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java @@ -119,8 +119,8 @@ public Property getTargetCompatibility() { return targetCompatibility; } + @Classpath @InputFiles - @PathSensitive(PathSensitivity.NAME_ONLY) public abstract ConfigurableFileCollection getForbiddenAPIsClasspath(); @InputFile diff --git a/docs/changelog/107481.yaml b/docs/changelog/107481.yaml deleted file mode 100644 index 9e65b457c9ed6..0000000000000 --- a/docs/changelog/107481.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 107481 -summary: Block specific config files from being read after startup -area: Security -type: bug -issues: [] diff --git a/docs/changelog/107545.yaml b/docs/changelog/107545.yaml new file mode 100644 index 0000000000000..ad457cc5a533f --- /dev/null +++ b/docs/changelog/107545.yaml @@ -0,0 +1,6 @@ +pr: 107545 +summary: "ESQL: Union Types Support" +area: ES|QL +type: enhancement +issues: + - 100603 diff --git a/docs/changelog/108171.yaml b/docs/changelog/108171.yaml new file mode 100644 index 0000000000000..1ec17bb3e411d --- /dev/null +++ b/docs/changelog/108171.yaml @@ -0,0 +1,5 @@ +pr: 108171 +summary: "add Elastic-internal stable bridge api for use by Logstash" +area: Infra/Core +type: enhancement +issues: [] diff --git a/docs/changelog/108793.yaml b/docs/changelog/108793.yaml new file mode 100644 index 0000000000000..87668c8ee009b --- /dev/null +++ b/docs/changelog/108793.yaml @@ -0,0 +1,5 @@ +pr: 108793 +summary: Add `SparseVectorStats` +area: Search +type: enhancement +issues: [] diff --git a/docs/changelog/108931.yaml b/docs/changelog/108931.yaml deleted file mode 100644 index 520637c5928e7..0000000000000 --- a/docs/changelog/108931.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 108931 -summary: Guard systemd library lookup from unreadable directories -area: Infra/Core -type: bug -issues: [] diff --git a/docs/changelog/109025.yaml b/docs/changelog/109025.yaml new file mode 100644 index 0000000000000..38d19cab13d30 --- /dev/null +++ b/docs/changelog/109025.yaml @@ -0,0 +1,6 @@ +pr: 109025 +summary: Introduce a setting controlling the activation of the `logs` index mode in logs@settings +area: Logs +type: feature +issues: + - 108762 diff --git a/docs/changelog/109276.yaml b/docs/changelog/109276.yaml deleted file mode 100644 index d73e68e3c8f7b..0000000000000 --- a/docs/changelog/109276.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109276 -summary: Add remove index setting command -area: Infra/Settings -type: enhancement -issues: [] diff --git a/docs/changelog/109317.yaml b/docs/changelog/109317.yaml new file mode 100644 index 0000000000000..1d8595d99c2a6 --- /dev/null +++ b/docs/changelog/109317.yaml @@ -0,0 +1,13 @@ +pr: 109317 +summary: Add new int4 quantization to dense_vector +area: Search +type: feature +issues: [] +highlight: + title: Add new int4 quantization to dense_vector + body: |- + New int4 (half-byte) scalar quantization support via two knew index types: `int4_hnsw` and `int4_flat`. + This gives an 8x reduction from `float32` with some accuracy loss. In addition to less memory required, this + improves query and merge speed significantly when compared to raw vectors. + notable: true + diff --git a/docs/changelog/109320.yaml b/docs/changelog/109320.yaml deleted file mode 100644 index 84aff5b1d769d..0000000000000 --- a/docs/changelog/109320.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109320 -summary: Reset retryable index requests after failures -area: Machine Learning -type: bug -issues: [] diff --git a/docs/changelog/109357.yaml b/docs/changelog/109357.yaml deleted file mode 100644 index 17951882103b3..0000000000000 --- a/docs/changelog/109357.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109357 -summary: Fix task cancellation authz on fulfilling cluster -area: Authorization -type: bug -issues: [] diff --git a/docs/changelog/109359.yaml b/docs/changelog/109359.yaml new file mode 100644 index 0000000000000..37202eb5a28ec --- /dev/null +++ b/docs/changelog/109359.yaml @@ -0,0 +1,5 @@ +pr: 109359 +summary: Adding hamming distance function to painless for `dense_vector` fields +area: Vector Search +type: enhancement +issues: [] diff --git a/docs/changelog/109386.yaml b/docs/changelog/109386.yaml new file mode 100644 index 0000000000000..984ee96dde063 --- /dev/null +++ b/docs/changelog/109386.yaml @@ -0,0 +1,6 @@ +pr: 109386 +summary: "ESQL: `top_list` aggregation" +area: ES|QL +type: feature +issues: + - 109213 diff --git a/docs/changelog/109423.yaml b/docs/changelog/109423.yaml deleted file mode 100644 index 5f594ea482338..0000000000000 --- a/docs/changelog/109423.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109423 -summary: Correct how hex strings are handled when dynamically updating vector dims -area: Vector Search -type: bug -issues: [] diff --git a/docs/changelog/109440.yaml b/docs/changelog/109440.yaml deleted file mode 100644 index c1e9aef8110fc..0000000000000 --- a/docs/changelog/109440.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 109440 -summary: Fix task cancellation on remote cluster when original request fails -area: Network -type: bug -issues: [] diff --git a/docs/changelog/109462.yaml b/docs/changelog/109462.yaml new file mode 100644 index 0000000000000..a05f4a04e80ae --- /dev/null +++ b/docs/changelog/109462.yaml @@ -0,0 +1,6 @@ +pr: 109462 +summary: Add `wait_for_completion` parameter to delete snapshot request +area: Distributed +type: enhancement +issues: + - 101300 diff --git a/docs/changelog/109480.yaml b/docs/changelog/109480.yaml new file mode 100644 index 0000000000000..3a6f48e9bd840 --- /dev/null +++ b/docs/changelog/109480.yaml @@ -0,0 +1,5 @@ +pr: 109480 +summary: "[Connector API] Add claim sync job endpoint" +area: Application +type: feature +issues: [] diff --git a/docs/changelog/109487.yaml b/docs/changelog/109487.yaml new file mode 100644 index 0000000000000..c69c77203f12d --- /dev/null +++ b/docs/changelog/109487.yaml @@ -0,0 +1,5 @@ +pr: 109487 +summary: Start Trained Model Deployment API request query params now override body params +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/109506.yaml b/docs/changelog/109506.yaml new file mode 100644 index 0000000000000..3a7570ed0b93a --- /dev/null +++ b/docs/changelog/109506.yaml @@ -0,0 +1,6 @@ +pr: 109506 +summary: Support synthetic source for `scaled_float` and `unsigned_long` when `ignore_malformed` + is used +area: Mapping +type: enhancement +issues: [] diff --git a/docs/changelog/109534.yaml b/docs/changelog/109534.yaml new file mode 100644 index 0000000000000..c6eb520bb70a8 --- /dev/null +++ b/docs/changelog/109534.yaml @@ -0,0 +1,6 @@ +pr: 109534 +summary: Propagate accurate deployment timeout +area: Machine Learning +type: bug +issues: + - 109407 diff --git a/docs/changelog/109551.yaml b/docs/changelog/109551.yaml new file mode 100644 index 0000000000000..f4949669091d9 --- /dev/null +++ b/docs/changelog/109551.yaml @@ -0,0 +1,5 @@ +pr: 109551 +summary: Avoid `InferenceRunner` deadlock +area: Machine Learning +type: bug +issues: [] diff --git a/docs/changelog/109563.yaml b/docs/changelog/109563.yaml new file mode 100644 index 0000000000000..9099064b6b040 --- /dev/null +++ b/docs/changelog/109563.yaml @@ -0,0 +1,5 @@ +pr: 109563 +summary: Add allocation explain output for THROTTLING shards +area: Infra/Core +type: enhancement +issues: [] diff --git a/docs/changelog/109597.yaml b/docs/changelog/109597.yaml new file mode 100644 index 0000000000000..9b99df85da6a3 --- /dev/null +++ b/docs/changelog/109597.yaml @@ -0,0 +1,5 @@ +pr: 109597 +summary: Opt `scripted_metric` out of parallelization +area: Aggregations +type: feature +issues: [] diff --git a/docs/changelog/109603.yaml b/docs/changelog/109603.yaml new file mode 100644 index 0000000000000..2d6e8b94aa8d0 --- /dev/null +++ b/docs/changelog/109603.yaml @@ -0,0 +1,5 @@ +pr: 109603 +summary: Update translog `writeLocation` for `flushListener` after commit +area: Engine +type: enhancement +issues: [] diff --git a/docs/changelog/109606.yaml b/docs/changelog/109606.yaml new file mode 100644 index 0000000000000..6c9089c4c4fde --- /dev/null +++ b/docs/changelog/109606.yaml @@ -0,0 +1,5 @@ +pr: 109606 +summary: Avoid NPE if `users_roles` file does not exist +area: Authentication +type: bug +issues: [] diff --git a/docs/changelog/109632.yaml b/docs/changelog/109632.yaml new file mode 100644 index 0000000000000..6b04160bbdbec --- /dev/null +++ b/docs/changelog/109632.yaml @@ -0,0 +1,5 @@ +pr: 109632 +summary: Force execute inactive sink reaper +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/109634.yaml b/docs/changelog/109634.yaml new file mode 100644 index 0000000000000..4c6358578b6de --- /dev/null +++ b/docs/changelog/109634.yaml @@ -0,0 +1,5 @@ +pr: 109634 +summary: "[Query Rules] Require Enterprise License for Query Rules" +area: Relevance +type: enhancement +issues: [] diff --git a/docs/changelog/109651.yaml b/docs/changelog/109651.yaml new file mode 100644 index 0000000000000..982e6a5b536cc --- /dev/null +++ b/docs/changelog/109651.yaml @@ -0,0 +1,5 @@ +pr: 109651 +summary: Support synthetic source for `geo_point` when `ignore_malformed` is used +area: Mapping +type: enhancement +issues: [] diff --git a/docs/changelog/109653.yaml b/docs/changelog/109653.yaml new file mode 100644 index 0000000000000..665163ec2a91b --- /dev/null +++ b/docs/changelog/109653.yaml @@ -0,0 +1,5 @@ +pr: 109653 +summary: Handle the "JSON memory allocator bytes" field +area: Machine Learning +type: enhancement +issues: [] diff --git a/docs/changelog/109657.yaml b/docs/changelog/109657.yaml new file mode 100644 index 0000000000000..35b315b7568c9 --- /dev/null +++ b/docs/changelog/109657.yaml @@ -0,0 +1,5 @@ +pr: 109657 +summary: Track `RequestedRangeNotSatisfiedException` separately in S3 Metrics +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/109672.yaml b/docs/changelog/109672.yaml new file mode 100644 index 0000000000000..bb6532ab7accf --- /dev/null +++ b/docs/changelog/109672.yaml @@ -0,0 +1,5 @@ +pr: 109672 +summary: Log repo UUID at generation/registration time +area: Snapshot/Restore +type: enhancement +issues: [] diff --git a/docs/changelog/109695.yaml b/docs/changelog/109695.yaml new file mode 100644 index 0000000000000..f922b76412676 --- /dev/null +++ b/docs/changelog/109695.yaml @@ -0,0 +1,5 @@ +pr: 109695 +summary: Fix ESQL cancellation for exchange requests +area: ES|QL +type: bug +issues: [] diff --git a/docs/changelog/109717.yaml b/docs/changelog/109717.yaml new file mode 100644 index 0000000000000..326657ea4ce21 --- /dev/null +++ b/docs/changelog/109717.yaml @@ -0,0 +1,5 @@ +pr: 109717 +summary: Bump jackson version in modules:repository-azure +area: Snapshot/Restore +type: upgrade +issues: [] diff --git a/docs/changelog/109720.yaml b/docs/changelog/109720.yaml new file mode 100644 index 0000000000000..b029726c84427 --- /dev/null +++ b/docs/changelog/109720.yaml @@ -0,0 +1,5 @@ +pr: 109720 +summary: "DocsStats: Add human readable bytesize" +area: Stats +type: enhancement +issues: [] diff --git a/docs/changelog/109746.yaml b/docs/changelog/109746.yaml new file mode 100644 index 0000000000000..5360f545333ac --- /dev/null +++ b/docs/changelog/109746.yaml @@ -0,0 +1,6 @@ +pr: 109746 +summary: ES|QL Add primitive float support to the Compute Engine +area: ES|QL +type: enhancement +issues: + - 109178 diff --git a/docs/changelog/109779.yaml b/docs/changelog/109779.yaml new file mode 100644 index 0000000000000..4ccd8d475ec8d --- /dev/null +++ b/docs/changelog/109779.yaml @@ -0,0 +1,5 @@ +pr: 109779 +summary: Include component templates in retention validaiton +area: Data streams +type: bug +issues: [] diff --git a/docs/changelog/109781.yaml b/docs/changelog/109781.yaml new file mode 100644 index 0000000000000..df74645b53d84 --- /dev/null +++ b/docs/changelog/109781.yaml @@ -0,0 +1,5 @@ +pr: 109781 +summary: ES|QL Add primitive float variants of all aggregators to the compute engine +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/changelog/109794.yaml b/docs/changelog/109794.yaml new file mode 100644 index 0000000000000..d244c69a903ba --- /dev/null +++ b/docs/changelog/109794.yaml @@ -0,0 +1,5 @@ +pr: 109794 +summary: Provide document size reporter with `MapperService` +area: Infra/Metrics +type: bug +issues: [] diff --git a/docs/changelog/109824.yaml b/docs/changelog/109824.yaml new file mode 100644 index 0000000000000..987e8c0a8b1a2 --- /dev/null +++ b/docs/changelog/109824.yaml @@ -0,0 +1,6 @@ +pr: 109824 +summary: Check array size before returning array item in script doc values +area: Infra/Scripting +type: bug +issues: + - 104998 diff --git a/docs/changelog/109850.yaml b/docs/changelog/109850.yaml new file mode 100644 index 0000000000000..0f11318765aea --- /dev/null +++ b/docs/changelog/109850.yaml @@ -0,0 +1,5 @@ +pr: 109850 +summary: Ensure tasks preserve versions in `MasterService` +area: Cluster Coordination +type: bug +issues: [] diff --git a/docs/changelog/109882.yaml b/docs/changelog/109882.yaml new file mode 100644 index 0000000000000..0f0fed01c5a7a --- /dev/null +++ b/docs/changelog/109882.yaml @@ -0,0 +1,5 @@ +pr: 109882 +summary: Support synthetic source together with `ignore_malformed` in histogram fields +area: Mapping +type: enhancement +issues: [] diff --git a/docs/painless/painless-api-reference/painless-api-reference-score/index.asciidoc b/docs/painless/painless-api-reference/painless-api-reference-score/index.asciidoc index 775c0cc212426..4300a1c7efc66 100644 --- a/docs/painless/painless-api-reference/painless-api-reference-score/index.asciidoc +++ b/docs/painless/painless-api-reference/painless-api-reference-score/index.asciidoc @@ -23,6 +23,7 @@ The following methods are directly callable without a class/instance qualifier. * double dotProduct(Object *, String *) * double l1norm(Object *, String *) * double l2norm(Object *, String *) +* double hamming(Object *, String *) * double randomScore(int *) * double randomScore(int *, String *) * double saturation(double, double) diff --git a/docs/reference/cat/anomaly-detectors.asciidoc b/docs/reference/cat/anomaly-detectors.asciidoc index 607a88d1e1a5c..3416c256881af 100644 --- a/docs/reference/cat/anomaly-detectors.asciidoc +++ b/docs/reference/cat/anomaly-detectors.asciidoc @@ -7,9 +7,9 @@ [IMPORTANT] ==== -cat APIs are only intended for human consumption using the command line or {kib} -console. They are _not_ intended for use by applications. For application -consumption, use the +cat APIs are only intended for human consumption using the command line or {kib} +console. They are _not_ intended for use by applications. For application +consumption, use the <>. ==== @@ -137,7 +137,7 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=sparse-bucket-count] `forecasts.memory.avg`, `fmavg`, `forecastsMemoryAvg`::: The average memory usage in bytes for forecasts related to the {anomaly-job}. - + `forecasts.memory.max`, `fmmax`, `forecastsMemoryMax`::: The maximum memory usage in bytes for forecasts related to the {anomaly-job}. @@ -145,8 +145,8 @@ The maximum memory usage in bytes for forecasts related to the {anomaly-job}. The minimum memory usage in bytes for forecasts related to the {anomaly-job}. `forecasts.memory.total`, `fmt`, `forecastsMemoryTotal`::: -The total memory usage in bytes for forecasts related to the {anomaly-job}. - +The total memory usage in bytes for forecasts related to the {anomaly-job}. + `forecasts.records.avg`, `fravg`, `forecastsRecordsAvg`::: The average number of `model_forecast` documents written for forecasts related to the {anomaly-job}. @@ -161,8 +161,8 @@ to the {anomaly-job}. `forecasts.records.total`, `frt`, `forecastsRecordsTotal`::: The total number of `model_forecast` documents written for forecasts related to -the {anomaly-job}. - +the {anomaly-job}. + `forecasts.time.avg`, `ftavg`, `forecastsTimeAvg`::: The average runtime in milliseconds for forecasts related to the {anomaly-job}. @@ -198,7 +198,7 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-bytes-exceeded] `model.categorization_status`, `mcs`, `modelCategorizationStatus`::: include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorization-status] - + `model.categorized_doc_count`, `mcdc`, `modelCategorizedDocCount`::: include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=categorized-doc-count] @@ -221,6 +221,9 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-limit-anomaly-jobs] (Default) include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-memory-status] +`model.output_memory_allocator_bytes`, `momab`, `modelOutputMemoryAllocatorBytes`::: +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=output-memory-allocator-bytes] + `model.over_fields`, `mof`, `modelOverFields`::: include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-over-field-count] @@ -232,10 +235,10 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=rare-category-count] `model.timestamp`, `mt`, `modelTimestamp`::: include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=model-timestamp] - + `model.total_category_count`, `mtcc`, `modelTotalCategoryCount`::: include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=total-category-count] - + `node.address`, `na`, `nodeAddress`::: The network address of the node. + @@ -261,7 +264,7 @@ include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=open-time] `state`, `s`::: (Default) -include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=state-anomaly-job] +include::{es-ref-dir}/ml/ml-shared.asciidoc[tag=state-anomaly-job] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=help] diff --git a/docs/reference/cat/nodes.asciidoc b/docs/reference/cat/nodes.asciidoc index bfee57d1daad7..fc5b01f9234e3 100644 --- a/docs/reference/cat/nodes.asciidoc +++ b/docs/reference/cat/nodes.asciidoc @@ -1,5 +1,6 @@ [[cat-nodes]] === cat nodes API + ++++ cat nodes ++++ @@ -7,8 +8,9 @@ [IMPORTANT] ==== cat APIs are only intended for human consumption using the command line or {kib} -console. They are _not_ intended for use by applications. For application -consumption, use the <>. +console. +They are _not_ intended for use by applications. +For application consumption, use the <>. ==== Returns information about a cluster's nodes. @@ -32,13 +34,15 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=bytes] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] `full_id`:: -(Optional, Boolean) If `true`, return the full node ID. If `false`, return the -shortened node ID. Defaults to `false`. +(Optional, Boolean) If `true`, return the full node ID. +If `false`, return the shortened node ID. +Defaults to `false`. include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + -- -If you do not specify which columns to include, the API returns the default columns in the order listed below. If you explicitly specify one or more columns, it only returns the specified columns. +If you do not specify which columns to include, the API returns the default columns in the order listed below. +If you explicitly specify one or more columns, it only returns the specified columns. Valid columns are: @@ -58,7 +62,8 @@ Valid columns are: (Default) Used file descriptors percentage, such as `1`. `node.role`, `r`, `role`, `nodeRole`:: -(Default) Roles of the node. Returned values include +(Default) Roles of the node. +Returned values include `c` (cold node), `d` (data node), `f` (frozen node), @@ -73,12 +78,13 @@ Valid columns are: `w` (warm node), and `-` (coordinating node only). + -For example, `dim` indicates a master-eligible data and ingest node. See +For example, `dim` indicates a master-eligible data and ingest node. +See <>. `master`, `m`:: -(Default) Indicates whether the node is the elected master node. Returned values -include `*` (elected master) and `-` (not elected master). +(Default) Indicates whether the node is the elected master node. +Returned values include `*` (elected master) and `-` (not elected master). `name`, `n`:: (Default) Node name, such as `I8hydUG`. @@ -149,9 +155,6 @@ Node uptime, such as `17.3m`. `completion.size`, `cs`, `completionSize`:: Size of completion, such as `0b`. -`dense_vector.value_count`, `dvc`, `denseVectorCount`:: -Number of indexed dense vector. - `fielddata.memory_size`, `fm`, `fielddataMemory`:: Used fielddata cache memory, such as `0b`. @@ -306,8 +309,7 @@ Memory used by index writer, such as `18mb`. Memory used by version map, such as `1.0kb`. `segments.fixed_bitset_memory`, `sfbm`, `fixedBitsetMemory`:: -Memory used by fixed bit sets for nested object field types and type filters for -types referred in <> fields, such as `1.0kb`. +Memory used by fixed bit sets for nested object field types and type filters for types referred in <> fields, such as `1.0kb`. `suggest.current`, `suc`, `suggestCurrent`:: Number of current suggest operations, such as `0`. @@ -362,15 +364,13 @@ ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master // TESTRESPONSE[s/65 99 42/\\d+ \\d+ \\d+/] // TESTRESPONSE[s/dim/.+/ s/[*]/[*]/ s/mJw06l1/.+/ non_json] -The `ip`, `heap.percent`, `ram.percent`, `cpu`, and `load_*` columns provide the -IP addresses and performance information of each node. - -The `node.role`, `master`, and `name` columns provide information useful for -monitoring an entire cluster, particularly large ones. +The `ip`, `heap.percent`, `ram.percent`, `cpu`, and `load_*` columns provide the IP addresses and performance information of each node. +The `node.role`, `master`, and `name` columns provide information useful for monitoring an entire cluster, particularly large ones. [[cat-nodes-api-ex-headings]] ===== Example with explicit columns + The following API request returns the `id`, `ip`, `port`, `v` (version), and `m` (master) columns. diff --git a/docs/reference/cat/shards.asciidoc b/docs/reference/cat/shards.asciidoc index 74c017d86d8e8..a2f8541be4abc 100644 --- a/docs/reference/cat/shards.asciidoc +++ b/docs/reference/cat/shards.asciidoc @@ -1,22 +1,21 @@ [[cat-shards]] === cat shards API + ++++ cat shards ++++ [IMPORTANT] ==== -cat APIs are only intended for human consumption using the command line or {kib} -console. They are _not_ intended for use by applications. +cat APIs are only intended for human consumption using the command line or {kib} +console. +They are _not_ intended for use by applications. ==== -The `shards` command is the detailed view of what nodes contain which -shards. It will tell you if it's a primary or replica, the number of -docs, the bytes it takes on disk, and the node where it's located. - -For data streams, the API returns information about the stream's backing -indices. +The `shards` command is the detailed view of what nodes contain which shards. +It will tell you if it's a primary or replica, the number of docs, the bytes it takes on disk, and the node where it's located. +For data streams, the API returns information about the stream's backing indices. [[cat-shards-api-request]] ==== {api-request-title} @@ -29,17 +28,17 @@ indices. ==== {api-prereq-title} * If the {es} {security-features} are enabled, you must have the `monitor` or -`manage` <> to use this API. You must -also have the `monitor` or `manage` <> +`manage` <> to use this API. +You must also have the `monitor` or `manage` <> for any data stream, index, or alias you retrieve. [[cat-shards-path-params]] ==== {api-path-parms-title} ``:: -(Optional, string) Comma-separated list of data streams, indices, and aliases -used to limit the request. Supports wildcards (`*`). To target all data streams -and indices, omit this parameter or use `*` or `_all`. +(Optional, string) Comma-separated list of data streams, indices, and aliases used to limit the request. +Supports wildcards (`*`). +To target all data streams and indices, omit this parameter or use `*` or `_all`. [[cat-shards-query-params]] ==== {api-query-parms-title} @@ -51,9 +50,8 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=http-format] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-h] + -- -If you do not specify which columns to include, the API returns the default -columns in the order listed below. If you explicitly specify one or more -columns, it only returns the specified columns. +If you do not specify which columns to include, the API returns the default columns in the order listed below. +If you explicitly specify one or more columns, it only returns the specified columns. Valid columns are: @@ -64,10 +62,12 @@ Valid columns are: (Default) Name of the shard. `prirep`, `p`, `pr`, `primaryOrReplica`:: -(Default) Shard type. Returned values are `primary` or `replica`. +(Default) Shard type. +Returned values are `primary` or `replica`. `state`, `st`:: -(Default) State of the shard. Returned values are: +(Default) State of the shard. +Returned values are: + * `INITIALIZING`: The shard is recovering from a peer shard or gateway. * `RELOCATING`: The shard is relocating. @@ -81,7 +81,8 @@ Valid columns are: (Default) Disk space used by the shard, such as `5kb`. `dataset.size`:: -(Default) Disk space used by the shard's dataset, which may or may not be the size on disk, but includes space used by the shard on object storage. Reported as a size value such as `5kb`. +(Default) Disk space used by the shard's dataset, which may or may not be the size on disk, but includes space used by the shard on object storage. +Reported as a size value such as `5kb`. `ip`:: (Default) IP address of the node, such as `127.0.1.1`. @@ -96,7 +97,7 @@ Valid columns are: Size of completion, such as `0b`. `dense_vector.value_count`, `dvc`, `denseVectorCount`:: -Number of indexed dense vector. +Number of indexed dense vectors. `fielddata.memory_size`, `fm`, `fielddataMemory`:: Used fielddata cache memory, such as `0b`. @@ -231,8 +232,7 @@ Memory used by index writer, such as `18mb`. Memory used by version map, such as `1.0kb`. `segments.fixed_bitset_memory`, `sfbm`, `fixedBitsetMemory`:: -Memory used by fixed bit sets for nested object field types and type filters for -types referred in <> fields, such as `1.0kb`. +Memory used by fixed bit sets for nested object field types and type filters for types referred in <> fields, such as `1.0kb`. `seq_no.global_checkpoint`, `sqg`, `globalCheckpoint`:: Global checkpoint. @@ -243,6 +243,9 @@ Local checkpoint. `seq_no.max`, `sqm`, `maxSeqNo`:: Maximum sequence number. +`sparse_vector.value_count`, `svc`, `sparseVectorCount`:: +Number of indexed <>. + `suggest.current`, `suc`, `suggestCurrent`:: Number of current suggest operations, such as `0`. @@ -257,25 +260,23 @@ Sync ID of the shard. `unassigned.at`, `ua`:: Time at which the shard became unassigned in -{wikipedia}/List_of_UTC_time_offsets[Coordinated Universal -Time (UTC)]. +{wikipedia}/List_of_UTC_time_offsets[Coordinated Universal Time (UTC)]. `unassigned.details`, `ud`:: -Details about why the shard became unassigned. This does not explain why the -shard is currently unassigned. To understand why a shard is not assigned, use -the <> API. +Details about why the shard became unassigned. +This does not explain why the shard is currently unassigned. +To understand why a shard is not assigned, use the <> API. `unassigned.for`, `uf`:: Time at which the shard was requested to be unassigned in -{wikipedia}/List_of_UTC_time_offsets[Coordinated Universal -Time (UTC)]. +{wikipedia}/List_of_UTC_time_offsets[Coordinated Universal Time (UTC)]. [[reason-unassigned]] `unassigned.reason`, `ur`:: Indicates the reason for the last change to the state of this unassigned shard. -This does not explain why the shard is currently unassigned. To understand why -a shard is not assigned, use the <> API. Returned -values include: +This does not explain why the shard is currently unassigned. +To understand why a shard is not assigned, use the <> API. +Returned values include: + * `ALLOCATION_FAILED`: Unassigned as a result of a failed allocation of the shard. * `CLUSTER_RECOVERED`: Unassigned as a result of a full cluster recovery. @@ -288,7 +289,7 @@ values include: * `MANUAL_ALLOCATION`: The shard's allocation was last modified by the <> API. * `NEW_INDEX_RESTORED`: Unassigned as a result of restoring into a new index. * `NODE_LEFT`: Unassigned as a result of the node hosting it leaving the cluster. -* `NODE_RESTARTING`: Similar to `NODE_LEFT`, except that the node was registered as restarting using the <>. +* `NODE_RESTARTING`: Similar to `NODE_LEFT`, except that the node was registered as restarting using the <>. * `PRIMARY_FAILED`: The shard was initializing as a replica, but the primary shard failed before the initialization completed. * `REALLOCATED_REPLICA`: A better replica location is identified and causes the existing replica allocation to be cancelled. * `REINITIALIZED`: When a shard moves from started back to initializing. @@ -307,7 +308,6 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=time] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cat-v] - [[cat-shards-api-example]] ==== {api-examples-title} @@ -337,8 +337,7 @@ my-index-000001 0 p STARTED 3014 31.1mb 192.168.56.10 H5dfFeA If your cluster has many shards, you can use a wildcard pattern in the `` path parameter to limit the API request. -The following request returns information for any data streams or indices -beginning with `my-index-`. +The following request returns information for any data streams or indices beginning with `my-index-`. [source,console] --------------------------------------------------------------------------- @@ -375,8 +374,7 @@ my-index-000001 0 p RELOCATING 3014 31.1mb 192.168.56.10 H5dfFeA -> -> 192.168.5 --------------------------------------------------------------------------- // TESTRESPONSE[non_json] -The `RELOCATING` value in `state` column indicates the index shard is -relocating. +The `RELOCATING` value in `state` column indicates the index shard is relocating. [[states]] ===== Example with a shard states @@ -401,9 +399,7 @@ my-index-000001 0 r INITIALIZING 0 14.3mb 192.168.56.30 bGG90GE ===== Example with reasons for unassigned shards -The following request returns the `unassigned.reason` column, which indicates -why a shard is unassigned. - +The following request returns the `unassigned.reason` column, which indicates why a shard is unassigned. [source,console] --------------------------------------------------------------------------- diff --git a/docs/reference/cluster/nodes-stats.asciidoc b/docs/reference/cluster/nodes-stats.asciidoc index 59cb7167028c8..084ff471367ce 100644 --- a/docs/reference/cluster/nodes-stats.asciidoc +++ b/docs/reference/cluster/nodes-stats.asciidoc @@ -1,5 +1,6 @@ [[cluster-nodes-stats]] === Nodes stats API + ++++ Nodes stats ++++ @@ -32,104 +33,96 @@ Returns cluster nodes statistics. You can use the cluster nodes stats API to retrieve statistics for nodes in a cluster. - All the nodes selective options are explained <>. -By default, all stats are returned. You can limit the returned information by -using metrics. +By default, all stats are returned. +You can limit the returned information by using metrics. [[cluster-nodes-stats-api-path-params]] ==== {api-path-parms-title} - ``:: - (Optional, string) Limits the information returned to the specific metrics. - A comma-separated list of the following options: +(Optional, string) Limits the information returned to the specific metrics. +A comma-separated list of the following options: + -- - `adaptive_selection`:: - Statistics about <>. +`adaptive_selection`:: +Statistics about <>. - `allocations`:: - Statistics about allocated shards +`allocations`:: +Statistics about allocated shards - `breaker`:: - Statistics about the field data circuit breaker. +`breaker`:: +Statistics about the field data circuit breaker. - `discovery`:: - Statistics about the discovery. +`discovery`:: +Statistics about the discovery. - `fs`:: - File system information, data path, free disk space, read/write - stats. +`fs`:: +File system information, data path, free disk space, read/write stats. - `http`:: - HTTP connection information. +`http`:: +HTTP connection information. - `indexing_pressure`:: - Statistics about the node's indexing load and related rejections. +`indexing_pressure`:: +Statistics about the node's indexing load and related rejections. - `indices`:: - Indices stats about size, document count, indexing and deletion times, - search times, field cache size, merges and flushes. +`indices`:: +Indices stats about size, document count, indexing and deletion times, search times, field cache size, merges and flushes. - `ingest`:: - Statistics about ingest preprocessing. +`ingest`:: +Statistics about ingest preprocessing. - `jvm`:: - JVM stats, memory pool information, garbage collection, buffer - pools, number of loaded/unloaded classes. +`jvm`:: +JVM stats, memory pool information, garbage collection, buffer pools, number of loaded/unloaded classes. - `os`:: - Operating system stats, load average, mem, swap. +`os`:: +Operating system stats, load average, mem, swap. - `process`:: - Process statistics, memory consumption, cpu usage, open - file descriptors. +`process`:: +Process statistics, memory consumption, cpu usage, open file descriptors. - `repositories`:: - Statistics about snapshot repositories. +`repositories`:: +Statistics about snapshot repositories. - `thread_pool`:: - Statistics about each thread pool, including current size, queue and - rejected tasks. +`thread_pool`:: +Statistics about each thread pool, including current size, queue and rejected tasks. - `transport`:: - Transport statistics about sent and received bytes in cluster - communication. +`transport`:: +Transport statistics about sent and received bytes in cluster communication. -- ``:: - (Optional, string) Limit the information returned for `indices` metric to - the specific index metrics. It can be used only if `indices` (or `all`) - metric is specified. Supported metrics are: +(Optional, string) Limit the information returned for `indices` metric to the specific index metrics. +It can be used only if `indices` (or `all`) metric is specified. +Supported metrics are: + -- - * `bulk` - * `completion` - * `docs` - * `fielddata` - * `flush` - * `get` - * `indexing` - * `mappings` - * `merge` - * `query_cache` - * `recovery` - * `refresh` - * `request_cache` - * `search` - * `segments` - * `shard_stats` - * `store` - * `translog` - * `warmer` - * `dense_vector` +* `bulk` +* `completion` +* `docs` +* `fielddata` +* `flush` +* `get` +* `indexing` +* `mappings` +* `merge` +* `query_cache` +* `recovery` +* `refresh` +* `request_cache` +* `search` +* `segments` +* `shard_stats` +* `store` +* `translog` +* `warmer` +* `dense_vector` +* `sparse_vector` -- include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-id] - [[cluster-nodes-stats-api-query-params]] ==== {api-query-parms-title} @@ -144,8 +137,8 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=groups] include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=level] `types`:: - (Optional, string) A comma-separated list of document types for the - `indexing` index metric. +(Optional, string) A comma-separated list of document types for the +`indexing` index metric. include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=timeout-nodes-request] @@ -158,86 +151,73 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=include-unloaded-segmen ==== {api-response-body-title} `_nodes`:: -(object) -Contains statistics about the number of nodes selected by the request. +(object) Contains statistics about the number of nodes selected by the request. + .Properties of `_nodes` [%collapsible%open] ==== `total`:: -(integer) -Total number of nodes selected by the request. +(integer) Total number of nodes selected by the request. `successful`:: -(integer) -Number of nodes that responded successfully to the request. +(integer) Number of nodes that responded successfully to the request. `failed`:: -(integer) -Number of nodes that rejected the request or failed to respond. If this value -is not `0`, a reason for the rejection or failure is included in the response. +(integer) Number of nodes that rejected the request or failed to respond. +If this value is not `0`, a reason for the rejection or failure is included in the response. + ==== `cluster_name`:: -(string) -Name of the cluster. Based on the <> setting. +(string) Name of the cluster. +Based on the <> setting. `nodes`:: -(object) -Contains statistics for the nodes selected by the request. +(object) Contains statistics for the nodes selected by the request. + .Properties of `nodes` [%collapsible%open] ==== + ``:: -(object) -Contains statistics for the node. +(object) Contains statistics for the node. + .Properties of `` [%collapsible%open] ===== `timestamp`:: -(integer) -Time the node stats were collected for this response. Recorded in milliseconds -since the {wikipedia}/Unix_time[Unix Epoch]. +(integer) Time the node stats were collected for this response. +Recorded in milliseconds since the {wikipedia}/Unix_time[Unix Epoch]. `name`:: -(string) -Human-readable identifier for the node. Based on the <> setting. +(string) Human-readable identifier for the node. +Based on the <> setting. `transport_address`:: -(string) -Host and port for the <>, used for internal -communication between nodes in a cluster. +(string) Host and port for the <>, used for internal communication between nodes in a cluster. `host`:: -(string) -Network host for the node, based on the <> setting. +(string) Network host for the node, based on the <> setting. `ip`:: -(string) -IP address and port for the node. +(string) IP address and port for the node. `roles`:: -(array of strings) -Roles assigned to the node. See <>. +(array of strings) Roles assigned to the node. +See <>. `attributes`:: -(object) -Contains a list of attributes for the node. +(object) Contains a list of attributes for the node. [[cluster-nodes-stats-api-response-body-indices]] `indices`:: -(object) -Contains statistics about indices with shards assigned to the node. +(object) Contains statistics about indices with shards assigned to the node. + .Properties of `indices` [%collapsible%open] ====== `docs`:: -(object) -Contains statistics about documents across all primary shards assigned to the -node. +(object) Contains statistics about documents across all primary shards assigned to the node. + .Properties of `docs` [%collapsible%open] @@ -249,1759 +229,1415 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-count] `deleted`:: (integer) include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=docs-deleted] + ======= `store`:: -(object) -Contains statistics about the size of shards assigned to the node. +(object) Contains statistics about the size of shards assigned to the node. + .Properties of `store` [%collapsible%open] ======= + `size`:: -(<>) -Total size of all shards assigned to the node. +(<>) Total size of all shards assigned to the node. `size_in_bytes`:: -(integer) -Total size, in bytes, of all shards assigned to the node. +(integer) Total size, in bytes, of all shards assigned to the node. `total_data_set_size`:: -(<>) -Total data set size of all shards assigned to the node. -This includes the size of shards not stored fully on the node, such as the -cache for <>. +(<>) Total data set size of all shards assigned to the node. +This includes the size of shards not stored fully on the node, such as the cache for <>. `total_data_set_size_in_bytes`:: -(integer) -Total data set size, in bytes, of all shards assigned to the node. -This includes the size of shards not stored fully on the node, such as the -cache for <>. +(integer) Total data set size, in bytes, of all shards assigned to the node. +This includes the size of shards not stored fully on the node, such as the cache for <>. `reserved`:: -(<>) -A prediction of how much larger the shard stores on this node will eventually -grow due to ongoing peer recoveries, restoring snapshots, and similar -activities. A value of `-1b` indicates that this is not available. +(<>) A prediction of how much larger the shard stores on this node will eventually grow due to ongoing peer recoveries, restoring snapshots, and similar activities. +A value of `-1b` indicates that this is not available. `reserved_in_bytes`:: -(integer) -A prediction, in bytes, of how much larger the shard stores on this node will -eventually grow due to ongoing peer recoveries, restoring snapshots, and -similar activities. A value of `-1` indicates that this is not available. +(integer) A prediction, in bytes, of how much larger the shard stores on this node will eventually grow due to ongoing peer recoveries, restoring snapshots, and similar activities. +A value of `-1` indicates that this is not available. + ======= `indexing`:: -(object) -Contains statistics about indexing operations for the node. +(object) Contains statistics about indexing operations for the node. + .Properties of `indexing` [%collapsible%open] ======= + `index_total`:: -(integer) -Total number of indexing operations. +(integer) Total number of indexing operations. `index_time`:: -(<>) -Total time spent performing indexing operations. +(<>) Total time spent performing indexing operations. `index_time_in_millis`:: -(integer) -Total time in milliseconds -spent performing indexing operations. +(integer) Total time in milliseconds spent performing indexing operations. `index_current`:: -(integer) -Number of indexing operations currently running. +(integer) Number of indexing operations currently running. `index_failed`:: -(integer) -Number of failed indexing operations. +(integer) Number of failed indexing operations. `delete_total`:: -(integer) -Total number of deletion operations. +(integer) Total number of deletion operations. `delete_time`:: -(<>) -Time spent performing deletion operations. +(<>) Time spent performing deletion operations. `delete_time_in_millis`:: -(integer) -Time in milliseconds -spent performing deletion operations. +(integer) Time in milliseconds spent performing deletion operations. `delete_current`:: -(integer) -Number of deletion operations currently running. +(integer) Number of deletion operations currently running. `noop_update_total`:: -(integer) -Total number of noop operations. +(integer) Total number of noop operations. `is_throttled`:: -(Boolean) -Number of times -operations were throttled. +(Boolean) Number of times operations were throttled. `throttle_time`:: -(<>) -Total time spent throttling operations. +(<>) Total time spent throttling operations. `throttle_time_in_millis`:: -(integer) -Total time in milliseconds -spent throttling operations. +(integer) Total time in milliseconds spent throttling operations. `write_load`:: -(double) -Average number of write threads used while indexing documents. +(double) Average number of write threads used while indexing documents. + ======= `get`:: -(object) -Contains statistics about get operations for the node. +(object) Contains statistics about get operations for the node. + .Properties of `get` [%collapsible%open] ======= + `total`:: -(integer) -Total number of get operations. +(integer) Total number of get operations. `getTime`:: -(<>) -Time spent performing get operations. +(<>) Time spent performing get operations. `time_in_millis`:: -(integer) -Time in milliseconds -spent performing get operations. +(integer) Time in milliseconds spent performing get operations. `exists_total`:: -(integer) -Total number of successful get operations. +(integer) Total number of successful get operations. `exists_time`:: -(<>) -Time spent performing successful get operations. +(<>) Time spent performing successful get operations. `exists_time_in_millis`:: -(integer) -Time in milliseconds -spent performing successful get operations. +(integer) Time in milliseconds spent performing successful get operations. `missing_total`:: -(integer) -Total number of failed get operations. +(integer) Total number of failed get operations. `missing_time`:: -(<>) -Time spent performing failed get operations. +(<>) Time spent performing failed get operations. `missing_time_in_millis`:: -(integer) -Time in milliseconds -spent performing failed get operations. +(integer) Time in milliseconds spent performing failed get operations. `current`:: -(integer) -Number of get operations currently running. +(integer) Number of get operations currently running. + ======= `search`:: -(object) -Contains statistics about search operations for the node. +(object) Contains statistics about search operations for the node. + .Properties of `search` [%collapsible%open] ======= + `open_contexts`:: -(integer) -Number of open search contexts. +(integer) Number of open search contexts. `query_total`:: -(integer) -Total number of query operations. +(integer) Total number of query operations. `query_time`:: -(<>) -Time spent performing query operations. +(<>) Time spent performing query operations. `query_time_in_millis`:: -(integer) -Time in milliseconds -spent performing query operations. +(integer) Time in milliseconds spent performing query operations. `query_current`:: -(integer) -Number of query operations currently running. +(integer) Number of query operations currently running. `fetch_total`:: -(integer) -Total number of fetch operations. +(integer) Total number of fetch operations. `fetch_time`:: -(<>) -Time spent performing fetch operations. +(<>) Time spent performing fetch operations. `fetch_time_in_millis`:: -(integer) -Time in milliseconds -spent performing fetch operations. +(integer) Time in milliseconds spent performing fetch operations. `fetch_current`:: -(integer) -Number of fetch operations currently running. +(integer) Number of fetch operations currently running. `scroll_total`:: -(integer) -Total number of scroll operations. +(integer) Total number of scroll operations. `scroll_time`:: -(<>) -Time spent performing scroll operations. +(<>) Time spent performing scroll operations. `scroll_time_in_millis`:: -(integer) -Time in milliseconds -spent performing scroll operations. +(integer) Time in milliseconds spent performing scroll operations. `scroll_current`:: -(integer) -Number of scroll operations currently running. +(integer) Number of scroll operations currently running. `suggest_total`:: -(integer) -Total number of suggest operations. +(integer) Total number of suggest operations. `suggest_time`:: -(<>) -Time spent performing suggest operations. +(<>) Time spent performing suggest operations. `suggest_time_in_millis`:: -(integer) -Time in milliseconds -spent performing suggest operations. +(integer) Time in milliseconds spent performing suggest operations. `suggest_current`:: -(integer) -Number of suggest operations currently running. +(integer) Number of suggest operations currently running. + ======= `merges`:: -(object) -Contains statistics about merge operations for the node. +(object) Contains statistics about merge operations for the node. + .Properties of `merges` [%collapsible%open] ======= + `current`:: -(integer) -Number of merge operations currently running. +(integer) Number of merge operations currently running. `current_docs`:: -(integer) -Number of document merges currently running. +(integer) Number of document merges currently running. `current_size`:: -(<>) -Memory used performing current document merges. +(<>) Memory used performing current document merges. `current_size_in_bytes`:: -(integer) -Memory, in bytes, used performing current document merges. +(integer) Memory, in bytes, used performing current document merges. `total`:: -(integer) -Total number of merge operations. +(integer) Total number of merge operations. `total_time`:: -(<>) -Total time spent performing merge operations. +(<>) Total time spent performing merge operations. `total_time_in_millis`:: -(integer) -Total time in milliseconds -spent performing merge operations. +(integer) Total time in milliseconds spent performing merge operations. `total_docs`:: -(integer) -Total number of merged documents. +(integer) Total number of merged documents. `total_size`:: -(<>) -Total size of document merges. +(<>) Total size of document merges. `total_size_in_bytes`:: -(integer) -Total size of document merges in bytes. +(integer) Total size of document merges in bytes. `total_stopped_time`:: -(<>) -Total time spent stopping merge operations. +(<>) Total time spent stopping merge operations. `total_stopped_time_in_millis`:: -(integer) -Total time in milliseconds -spent stopping merge operations. +(integer) Total time in milliseconds spent stopping merge operations. `total_throttled_time`:: -(<>) -Total time spent throttling merge operations. +(<>) Total time spent throttling merge operations. `total_throttled_time_in_millis`:: -(integer) -Total time in milliseconds -spent throttling merge operations. +(integer) Total time in milliseconds spent throttling merge operations. `total_auto_throttle`:: -(<>) -Size of automatically throttled merge operations. +(<>) Size of automatically throttled merge operations. `total_auto_throttle_in_bytes`:: -(integer) -Size, in bytes, of automatically throttled merge operations. +(integer) Size, in bytes, of automatically throttled merge operations. + ======= `refresh`:: -(object) -Contains statistics about refresh operations for the node. +(object) Contains statistics about refresh operations for the node. + .Properties of `refresh` [%collapsible%open] ======= + `total`:: -(integer) -Total number of refresh operations. +(integer) Total number of refresh operations. `total_time`:: -(<>) -Total time spent performing refresh operations. +(<>) Total time spent performing refresh operations. `total_time_in_millis`:: -(integer) -Total time in milliseconds -spent performing refresh operations. +(integer) Total time in milliseconds spent performing refresh operations. `external_total`:: -(integer) -Total number of external refresh operations. +(integer) Total number of external refresh operations. `external_total_time`:: -(<>) -Total time spent performing external operations. +(<>) Total time spent performing external operations. `external_total_time_in_millis`:: -(integer) -Total time in milliseconds -spent performing external operations. +(integer) Total time in milliseconds spent performing external operations. `listeners`:: -(integer) -Number of refresh listeners. +(integer) Number of refresh listeners. + ======= `flush`:: -(object) -Contains statistics about flush operations for the node. +(object) Contains statistics about flush operations for the node. + .Properties of `flush` [%collapsible%open] ======= + `total`:: -(integer) -Number of flush operations. +(integer) Number of flush operations. `periodic`:: -(integer) -Number of flush periodic operations. +(integer) Number of flush periodic operations. `total_time`:: -(<>) -Total time spent performing flush operations. +(<>) Total time spent performing flush operations. `total_time_in_millis`:: -(integer) -Total time in milliseconds -spent performing flush operations. +(integer) Total time in milliseconds spent performing flush operations. ======= `warmer`:: -(object) -Contains statistics about index warming operations for the node. +(object) Contains statistics about index warming operations for the node. + .Properties of `warmer` [%collapsible%open] ======= + `current`:: -(integer) -Number of active index warmers. +(integer) Number of active index warmers. `total`:: -(integer) -Total number of index warmers. +(integer) Total number of index warmers. `total_time`:: -(<>) -Total time spent performing index warming operations. +(<>) Total time spent performing index warming operations. `total_time_in_millis`:: -(integer) -Total time in milliseconds -spent performing index warming operations. +(integer) Total time in milliseconds spent performing index warming operations. + ======= `query_cache`:: -(object) -Contains statistics about the query cache across all shards assigned to the -node. +(object) Contains statistics about the query cache across all shards assigned to the node. + .Properties of `query_cache` [%collapsible%open] ======= + `memory_size`:: -(<>) -Total amount of memory used for the query cache across all shards assigned to -the node. +(<>) Total amount of memory used for the query cache across all shards assigned to the node. `memory_size_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for the query cache across all shards -assigned to the node. +(integer) Total amount of memory, in bytes, used for the query cache across all shards assigned to the node. `total_count`:: -(integer) -Total count of hits, misses, and cached queries -in the query cache. +(integer) Total count of hits, misses, and cached queries in the query cache. `hit_count`:: -(integer) -Number of query cache hits. +(integer) Number of query cache hits. `miss_count`:: -(integer) -Number of query cache misses. +(integer) Number of query cache misses. `cache_size`:: -(integer) -Current number of cached queries. +(integer) Current number of cached queries. `cache_count`:: -(integer) -Total number of all queries that have been cached. +(integer) Total number of all queries that have been cached. `evictions`:: -(integer) -Number of query cache evictions. +(integer) Number of query cache evictions. + ======= `fielddata`:: -(object) -Contains statistics about the field data cache across all shards -assigned to the node. +(object) Contains statistics about the field data cache across all shards assigned to the node. + .Properties of `fielddata` [%collapsible%open] ======= + `memory_size`:: -(<>) -Total amount of memory used for the field data cache across all shards -assigned to the node. +(<>) Total amount of memory used for the field data cache across all shards assigned to the node. `memory_size_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for the field data cache across all -shards assigned to the node. +(integer) Total amount of memory, in bytes, used for the field data cache across all shards assigned to the node. `evictions`:: -(integer) -Number of fielddata evictions. +(integer) Number of fielddata evictions. + ======= `completion`:: -(object) -Contains statistics about completions across all shards assigned to the node. +(object) Contains statistics about completions across all shards assigned to the node. + .Properties of `completion` [%collapsible%open] ======= + `size`:: -(<>) -Total amount of memory used for completion across all shards assigned to -the node. +(<>) Total amount of memory used for completion across all shards assigned to the node. `size_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for completion across all shards assigned -to the node. +(integer) Total amount of memory, in bytes, used for completion across all shards assigned to the node. + ======= `segments`:: -(object) -Contains statistics about segments across all shards assigned to the node. +(object) Contains statistics about segments across all shards assigned to the node. + .Properties of `segments` [%collapsible%open] ======= + `count`:: -(integer) -Number of segments. +(integer) Number of segments. `memory`:: -(<>) -Total amount of memory used for segments across all shards assigned to the -node. +(<>) Total amount of memory used for segments across all shards assigned to the node. `memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for segments across all shards assigned -to the node. +(integer) Total amount of memory, in bytes, used for segments across all shards assigned to the node. `terms_memory`:: -(<>) -Total amount of memory used for terms across all shards assigned to the node. +(<>) Total amount of memory used for terms across all shards assigned to the node. `terms_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for terms across all shards assigned to -the node. +(integer) Total amount of memory, in bytes, used for terms across all shards assigned to the node. `stored_fields_memory`:: -(<>) -Total amount of memory used for stored fields across all shards assigned to -the node. +(<>) Total amount of memory used for stored fields across all shards assigned to the node. `stored_fields_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for stored fields across all shards -assigned to the node. +(integer) Total amount of memory, in bytes, used for stored fields across all shards assigned to the node. `term_vectors_memory`:: -(<>) -Total amount of memory used for term vectors across all shards assigned to -the node. +(<>) Total amount of memory used for term vectors across all shards assigned to the node. `term_vectors_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for term vectors across all shards -assigned to the node. +(integer) Total amount of memory, in bytes, used for term vectors across all shards assigned to the node. `norms_memory`:: -(<>) -Total amount of memory used for normalization factors across all shards assigned -to the node. +(<>) Total amount of memory used for normalization factors across all shards assigned to the node. `norms_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for normalization factors across all -shards assigned to the node. +(integer) Total amount of memory, in bytes, used for normalization factors across all shards assigned to the node. `points_memory`:: -(<>) -Total amount of memory used for points across all shards assigned to the node. +(<>) Total amount of memory used for points across all shards assigned to the node. `points_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for points across all shards assigned to -the node. +(integer) Total amount of memory, in bytes, used for points across all shards assigned to the node. `doc_values_memory`:: -(<>) -Total amount of memory used for doc values across all shards assigned to -the node. +(<>) Total amount of memory used for doc values across all shards assigned to the node. `doc_values_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used for doc values across all shards assigned -to the node. +(integer) Total amount of memory, in bytes, used for doc values across all shards assigned to the node. `index_writer_memory`:: -(<>) -Total amount of memory used by all index writers across all shards assigned to -the node. +(<>) Total amount of memory used by all index writers across all shards assigned to the node. `index_writer_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used by all index writers across all shards -assigned to the node. +(integer) Total amount of memory, in bytes, used by all index writers across all shards assigned to the node. `version_map_memory`:: -(<>) -Total amount of memory used by all version maps across all shards assigned to -the node. +(<>) Total amount of memory used by all version maps across all shards assigned to the node. `version_map_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used by all version maps across all shards -assigned to the node. +(integer) Total amount of memory, in bytes, used by all version maps across all shards assigned to the node. `fixed_bit_set`:: -(<>) -Total amount of memory used by fixed bit sets across all shards assigned to -the node. +(<>) Total amount of memory used by fixed bit sets across all shards assigned to the node. + -Fixed bit sets are used for nested object field types and -type filters for <> fields. +Fixed bit sets are used for nested object field types and type filters for <> fields. `fixed_bit_set_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used by fixed bit sets across all shards -assigned to the node. +(integer) Total amount of memory, in bytes, used by fixed bit sets across all shards assigned to the node. + -Fixed bit sets are used for nested object field types and -type filters for <> fields. +Fixed bit sets are used for nested object field types and type filters for <> fields. `max_unsafe_auto_id_timestamp`:: -(integer) -Time of the most recently retried indexing request. Recorded in milliseconds -since the {wikipedia}/Unix_time[Unix Epoch]. +(integer) Time of the most recently retried indexing request. +Recorded in milliseconds since the {wikipedia}/Unix_time[Unix Epoch]. `file_sizes`:: -(object) -Contains statistics about the size of the segment file. +(object) Contains statistics about the size of the segment file. + .Properties of `file_sizes` [%collapsible%open] ======== `size`:: -(<>) -Size of the segment file. +(<>) Size of the segment file. `size_in_bytes`:: -(integer) -Size, in bytes, -of the segment file. +(integer) Size, in bytes, of the segment file. `description`:: -(string) -Description of the segment file. +(string) Description of the segment file. + ======== ======= `translog`:: -(object) -Contains statistics about transaction log operations for the node. +(object) Contains statistics about transaction log operations for the node. + .Properties of `translog` [%collapsible%open] ======= + `operations`:: -(integer) -Number of transaction log operations. +(integer) Number of transaction log operations. `size`:: -(<>) -Size of the transaction log. +(<>) Size of the transaction log. `size_in_bytes`:: -(integer) -Size, in bytes, of the transaction log. +(integer) Size, in bytes, of the transaction log. `uncommitted_operations`:: -(integer) -Number of uncommitted transaction log operations. +(integer) Number of uncommitted transaction log operations. `uncommitted_size`:: -(<>) -Size of uncommitted transaction log operations. +(<>) Size of uncommitted transaction log operations. `uncommitted_size_in_bytes`:: -(integer) -Size, in bytes, of uncommitted transaction log operations. +(integer) Size, in bytes, of uncommitted transaction log operations. `earliest_last_modified_age`:: -(integer) -Earliest last modified age -for the transaction log. +(integer) Earliest last modified age for the transaction log. + ======= `request_cache`:: -(object) -Contains statistics about the request cache across all shards assigned to the -node. +(object) Contains statistics about the request cache across all shards assigned to the node. + .Properties of `request_cache` [%collapsible%open] ======= + `memory_size`:: -(<>) -Memory used by the request cache. +(<>) Memory used by the request cache. `memory_size_in_bytes`:: -(integer) -Memory, in bytes, used by the request cache. +(integer) Memory, in bytes, used by the request cache. `evictions`:: -(integer) -Number of request cache operations. +(integer) Number of request cache operations. `hit_count`:: -(integer) -Number of request cache hits. +(integer) Number of request cache hits. `miss_count`:: -(integer) -Number of request cache misses. +(integer) Number of request cache misses. + ======= `recovery`:: -(object) -Contains statistics about recovery operations for the node. +(object) Contains statistics about recovery operations for the node. + .Properties of `recovery` [%collapsible%open] ======= + `current_as_source`:: -(integer) -Number of recoveries -that used an index shard as a source. +(integer) Number of recoveries that used an index shard as a source. `current_as_target`:: -(integer) -Number of recoveries -that used an index shard as a target. +(integer) Number of recoveries that used an index shard as a target. `throttle_time`:: -(<>) -Time by which recovery operations were delayed due to throttling. +(<>) Time by which recovery operations were delayed due to throttling. `throttle_time_in_millis`:: -(integer) -Time in milliseconds -recovery operations were delayed due to throttling. +(integer) Time in milliseconds recovery operations were delayed due to throttling. + ======= `shard_stats`:: -(object) -Contains statistics about all shards assigned to the node. +(object) Contains statistics about all shards assigned to the node. + .Properties of `shard_stats` [%collapsible%open] ======= + `total_count`:: -(integer) -The total number of shards assigned to the node. +(integer) The total number of shards assigned to the node. + ======= `mappings`:: -(object) -Contains statistics about the mappings for the node. -This is not shown for the `shards` level, since mappings may be -shared across the shards of an index on a node. +(object) Contains statistics about the mappings for the node. +This is not shown for the `shards` level, since mappings may be shared across the shards of an index on a node. + .Properties of `mappings` [%collapsible%open] ======= + `total_count`:: -(integer) -Number of mappings, including <> and <> fields. +(integer) Number of mappings, including <> and <> fields. `total_estimated_overhead`:: -(<>) -Estimated heap overhead of mappings on this node, which allows for 1kiB of heap for every mapped field. +(<>) Estimated heap overhead of mappings on this node, which allows for 1kiB of heap for every mapped field. `total_estimated_overhead_in_bytes`:: -(integer) -Estimated heap overhead, in bytes, of mappings on this node, which allows for 1kiB of heap for every mapped field. +(integer) Estimated heap overhead, in bytes, of mappings on this node, which allows for 1kiB of heap for every mapped field. + ======= `dense_vector`:: -(object) -Contains statistics about dense_vector across all shards assigned to the node. +(object) Contains statistics about dense_vector across all shards assigned to the node. + .Properties of `dense_vector` [%collapsible%open] ======= + `value_count`:: -(integer) -Total number of dense vector indexed across all shards assigned -to the node. +(integer) Total number of dense vector indexed across all shards assigned to the node. + +======= + +`sparse_vector`:: +(object) Contains statistics about sparse_vector across all shards assigned to the node. ++ +.Properties of `sparse_vector` +[%collapsible%open] ======= + +`value_count`:: +(integer) Total number of sparse vector indexed across all shards assigned to the node. + +======= + ====== [[cluster-nodes-stats-api-response-body-os]] `os`:: -(object) -Contains statistics about the operating system for the node. +(object) Contains statistics about the operating system for the node. + .Properties of `os` [%collapsible%open] ====== + `timestamp`:: -(integer) -Last time the operating system statistics were refreshed. Recorded in -milliseconds since the {wikipedia}/Unix_time[Unix Epoch]. +(integer) Last time the operating system statistics were refreshed. +Recorded in milliseconds since the {wikipedia}/Unix_time[Unix Epoch]. `cpu`:: -(object) -Contains statistics about CPU usage for the node. +(object) Contains statistics about CPU usage for the node. + .Properties of `cpu` [%collapsible%open] ======= + `percent`:: -(integer) -Recent CPU usage for the whole system, or `-1` if not supported. +(integer) Recent CPU usage for the whole system, or `-1` if not supported. `load_average`:: -(object) -Contains statistics about load averages on the system. +(object) Contains statistics about load averages on the system. + .Properties of `load_average` [%collapsible%open] ======== + `1m`:: -(float) -One-minute load average on the system (field is not present if one-minute load -average is not available). +(float) One-minute load average on the system (field is not present if one-minute load average is not available). `5m`:: -(float) -Five-minute load average on the system (field is not present if five-minute load -average is not available). +(float) Five-minute load average on the system (field is not present if five-minute load average is not available). `15m`:: -(float) -Fifteen-minute load average on the system (field is not present if -fifteen-minute load average is not available). +(float) Fifteen-minute load average on the system (field is not present if fifteen-minute load average is not available). + ======== ======= `mem`:: -(object) -Contains statistics about memory usage for the node. +(object) Contains statistics about memory usage for the node. + .Properties of `mem` [%collapsible%open] ======= + `total`:: -(<>) -Total amount of physical memory. +(<>) Total amount of physical memory. `total_in_bytes`:: -(integer) -Total amount of physical memory in bytes. +(integer) Total amount of physical memory in bytes. `adjusted_total`:: -(<>) -If the amount of physical memory has been overridden using the `es.total_memory_bytes` -system property then this reports the overridden value. Otherwise it reports the same -value as `total`. +(<>) If the amount of physical memory has been overridden using the `es.total_memory_bytes` +system property then this reports the overridden value. +Otherwise it reports the same value as `total`. `adjusted_total_in_bytes`:: -(integer) -If the amount of physical memory has been overridden using the `es.total_memory_bytes` -system property then this reports the overridden value in bytes. Otherwise it reports -the same value as `total_in_bytes`. +(integer) If the amount of physical memory has been overridden using the `es.total_memory_bytes` +system property then this reports the overridden value in bytes. +Otherwise it reports the same value as `total_in_bytes`. `free`:: -(<>) -Amount of free physical memory. +(<>) Amount of free physical memory. `free_in_bytes`:: -(integer) -Amount of free physical memory in bytes. +(integer) Amount of free physical memory in bytes. `used`:: -(<>) -Amount of used physical memory. +(<>) Amount of used physical memory. `used_in_bytes`:: -(integer) -Amount of used physical memory in bytes. +(integer) Amount of used physical memory in bytes. `free_percent`:: -(integer) -Percentage of free memory. +(integer) Percentage of free memory. `used_percent`:: -(integer) -Percentage of used memory. +(integer) Percentage of used memory. + ======= `swap`:: -(object) -Contains statistics about swap space for the node. +(object) Contains statistics about swap space for the node. + .Properties of `swap` [%collapsible%open] ======= + `total`:: -(<>) -Total amount of swap space. +(<>) Total amount of swap space. `total_in_bytes`:: -(integer) -Total amount of swap space in bytes. +(integer) Total amount of swap space in bytes. `free`:: -(<>) -Amount of free swap space. +(<>) Amount of free swap space. `free_in_bytes`:: -(integer) -Amount of free swap space in bytes. +(integer) Amount of free swap space in bytes. `used`:: -(<>) -Amount of used swap space. +(<>) Amount of used swap space. `used_in_bytes`:: -(integer) -Amount of used swap space in bytes. +(integer) Amount of used swap space in bytes. + ======= `cgroup` (Linux only):: -(object) -Contains cgroup statistics for the node. +(object) Contains cgroup statistics for the node. + -NOTE: For the cgroup stats to be visible, cgroups must be compiled into the -kernel, the `cpu` and `cpuacct` cgroup subsystems must be configured and stats -must be readable from `/sys/fs/cgroup/cpu` and `/sys/fs/cgroup/cpuacct`. +NOTE: For the cgroup stats to be visible, cgroups must be compiled into the kernel, the `cpu` and `cpuacct` cgroup subsystems must be configured and stats must be readable from `/sys/fs/cgroup/cpu` and `/sys/fs/cgroup/cpuacct`. + .Properties of `cgroup` [%collapsible%open] ======= `cpuacct` (Linux only):: -(object) -Contains statistics about `cpuacct` control group for the node. +(object) Contains statistics about `cpuacct` control group for the node. + .Properties of `cpuacct` [%collapsible%open] ======== + `control_group` (Linux only):: -(string) -The `cpuacct` control group to which the {es} process belongs. +(string) The `cpuacct` control group to which the {es} process belongs. `usage_nanos` (Linux only):: -(integer) -The total CPU time (in nanoseconds) consumed by all tasks in the same cgroup -as the {es} process. +(integer) The total CPU time (in nanoseconds) consumed by all tasks in the same cgroup as the {es} process. + ======== `cpu` (Linux only):: -(object) -Contains statistics about `cpu` control group for the node. +(object) Contains statistics about `cpu` control group for the node. + .Properties of `cpu` [%collapsible%open] ======== + `control_group` (Linux only):: -(string) -The `cpu` control group to which the {es} process belongs. +(string) The `cpu` control group to which the {es} process belongs. `cfs_period_micros` (Linux only):: -(integer) -The period of time (in microseconds) for how regularly all tasks in the same -cgroup as the {es} process should have their access to CPU resources -reallocated. +(integer) The period of time (in microseconds) for how regularly all tasks in the same cgroup as the {es} process should have their access to CPU resources reallocated. `cfs_quota_micros` (Linux only):: -(integer) -The total amount of time (in microseconds) for which all tasks in -the same cgroup as the {es} process can run during one period +(integer) The total amount of time (in microseconds) for which all tasks in the same cgroup as the {es} process can run during one period `cfs_period_micros`. `stat` (Linux only):: -(object) -Contains CPU statistics for the node. +(object) Contains CPU statistics for the node. + .Properties of `stat` [%collapsible%open] ========= `number_of_elapsed_periods` (Linux only):: -(integer) -The number of reporting periods (as specified by +(integer) The number of reporting periods (as specified by `cfs_period_micros`) that have elapsed. `number_of_times_throttled` (Linux only):: -(integer) -The number of times all tasks in the same cgroup as the {es} process have -been throttled. +(integer) The number of times all tasks in the same cgroup as the {es} process have been throttled. `time_throttled_nanos` (Linux only):: -(integer) -The total amount of time (in nanoseconds) for which all tasks in the same -cgroup as the {es} process have been throttled. +(integer) The total amount of time (in nanoseconds) for which all tasks in the same cgroup as the {es} process have been throttled. + ========= ======== `memory` (Linux only):: -(object) -Contains statistics about the `memory` control group for the node. +(object) Contains statistics about the `memory` control group for the node. + .Properties of `memory` [%collapsible%open] ======== + `control_group` (Linux only):: -(string) -The `memory` control group to which the {es} process belongs. +(string) The `memory` control group to which the {es} process belongs. `limit_in_bytes` (Linux only):: -(string) -The maximum amount of user memory (including file cache) allowed for all -tasks in the same cgroup as the {es} process. This value can be too big to -store in a `long`, so is returned as a string so that the value returned can -exactly match what the underlying operating system interface returns. Any -value that is too large to parse into a `long` almost certainly means no -limit has been set for the cgroup. +(string) The maximum amount of user memory (including file cache) allowed for all tasks in the same cgroup as the {es} process. +This value can be too big to store in a `long`, so is returned as a string so that the value returned can exactly match what the underlying operating system interface returns. +Any value that is too large to parse into a `long` almost certainly means no limit has been set for the cgroup. `usage_in_bytes` (Linux only):: -(string) -The total current memory usage by processes in the cgroup (in bytes) by all -tasks in the same cgroup as the {es} process. This value is stored as a -string for consistency with `limit_in_bytes`. +(string) The total current memory usage by processes in the cgroup (in bytes) by all tasks in the same cgroup as the {es} process. +This value is stored as a string for consistency with `limit_in_bytes`. + ======== ======= ====== [[cluster-nodes-stats-api-response-body-process]] `process`:: -(object) -Contains process statistics for the node. +(object) Contains process statistics for the node. + .Properties of `process` [%collapsible%open] ====== + `timestamp`:: -(integer) -Last time the statistics were refreshed. Recorded in milliseconds -since the {wikipedia}/Unix_time[Unix Epoch]. +(integer) Last time the statistics were refreshed. +Recorded in milliseconds since the {wikipedia}/Unix_time[Unix Epoch]. `open_file_descriptors`:: -(integer) -Number of opened file descriptors associated with the current or +(integer) Number of opened file descriptors associated with the current or `-1` if not supported. `max_file_descriptors`:: -(integer) -Maximum number of file descriptors allowed on the system, or `-1` if not -supported. +(integer) Maximum number of file descriptors allowed on the system, or `-1` if not supported. `cpu`:: -(object) -Contains CPU statistics for the node. +(object) Contains CPU statistics for the node. + .Properties of `cpu` [%collapsible%open] ======= + `percent`:: -(integer) -CPU usage in percent, or `-1` if not known at the time the stats are -computed. +(integer) CPU usage in percent, or `-1` if not known at the time the stats are computed. `total`:: -(<>) -CPU time used by the process on which the Java virtual machine is running. +(<>) CPU time used by the process on which the Java virtual machine is running. `total_in_millis`:: -(integer) -CPU time (in milliseconds) used by the process on which the Java virtual -machine is running, or `-1` if not supported. +(integer) CPU time (in milliseconds) used by the process on which the Java virtual machine is running, or `-1` if not supported. + ======= `mem`:: -(object) -Contains virtual memory statistics for the node. +(object) Contains virtual memory statistics for the node. + .Properties of `mem` [%collapsible%open] ======= + `total_virtual`:: -(<>) -Size of virtual memory that is guaranteed to be available to the -running process. +(<>) Size of virtual memory that is guaranteed to be available to the running process. `total_virtual_in_bytes`:: -(integer) -Size in bytes of virtual memory that is guaranteed to be available to the -running process. +(integer) Size in bytes of virtual memory that is guaranteed to be available to the running process. + ======= ====== [[cluster-nodes-stats-api-response-body-jvm]] `jvm`:: -(object) -Contains Java Virtual Machine (JVM) statistics for the node. +(object) Contains Java Virtual Machine (JVM) statistics for the node. + .Properties of `jvm` [%collapsible%open] ====== + `timestamp`:: -(integer) -Last time JVM statistics were refreshed. +(integer) Last time JVM statistics were refreshed. `uptime`:: -(<>) -Human-readable JVM uptime. Only returned if the +(<>) Human-readable JVM uptime. +Only returned if the <<_human_readable_output,`human`>> query parameter is `true`. `uptime_in_millis`:: -(integer) -JVM uptime in milliseconds. +(integer) JVM uptime in milliseconds. `mem`:: -(object) -Contains JVM memory usage statistics for the node. +(object) Contains JVM memory usage statistics for the node. + .Properties of `mem` [%collapsible%open] ======= + `heap_used`:: -(<>) -Memory currently in use by the heap. +(<>) Memory currently in use by the heap. `heap_used_in_bytes`:: -(integer) -Memory, in bytes, currently in use by the heap. +(integer) Memory, in bytes, currently in use by the heap. `heap_used_percent`:: -(integer) -Percentage of memory currently in use by the heap. +(integer) Percentage of memory currently in use by the heap. `heap_committed`:: -(<>) -Amount of memory available for use by the heap. +(<>) Amount of memory available for use by the heap. `heap_committed_in_bytes`:: -(integer) -Amount of memory, in bytes, available for use by the heap. +(integer) Amount of memory, in bytes, available for use by the heap. `heap_max`:: -(<>) -Maximum amount of memory available for use by the heap. +(<>) Maximum amount of memory available for use by the heap. `heap_max_in_bytes`:: -(integer) -Maximum amount of memory, in bytes, available for use by the heap. +(integer) Maximum amount of memory, in bytes, available for use by the heap. `non_heap_used`:: -(<>) -Non-heap memory used. +(<>) Non-heap memory used. `non_heap_used_in_bytes`:: -(integer) -Non-heap memory used, in bytes. +(integer) Non-heap memory used, in bytes. `non_heap_committed`:: -(<>) -Amount of non-heap memory available. +(<>) Amount of non-heap memory available. `non_heap_committed_in_bytes`:: -(integer) -Amount of non-heap memory available, in bytes. +(integer) Amount of non-heap memory available, in bytes. `pools`:: -(object) -Contains statistics about heap memory usage for the node. +(object) Contains statistics about heap memory usage for the node. + .Properties of `pools` [%collapsible%open] ======== `young`:: -(object) -Contains statistics about memory usage by the young generation heap for the -node. +(object) Contains statistics about memory usage by the young generation heap for the node. + .Properties of `young` [%collapsible%open] ========= + `used`:: -(<>) -Memory used by the young generation heap. +(<>) Memory used by the young generation heap. `used_in_bytes`:: -(integer) -Memory, in bytes, used by the young generation heap. +(integer) Memory, in bytes, used by the young generation heap. `max`:: -(<>) -Maximum amount of memory available for use by the young generation heap. +(<>) Maximum amount of memory available for use by the young generation heap. `max_in_bytes`:: -(integer) -Maximum amount of memory, in bytes, available for use by the young generation -heap. +(integer) Maximum amount of memory, in bytes, available for use by the young generation heap. `peak_used`:: -(<>) -Largest amount of memory historically used by the young generation heap. +(<>) Largest amount of memory historically used by the young generation heap. `peak_used_in_bytes`:: -(integer) -Largest amount of memory, in bytes, historically used by the young generation -heap. +(integer) Largest amount of memory, in bytes, historically used by the young generation heap. `peak_max`:: -(<>) -Largest amount of memory historically used by the young generation heap. +(<>) Largest amount of memory historically used by the young generation heap. `peak_max_in_bytes`:: -(integer) -Largest amount of memory, in bytes, historically used by the young generation -heap. +(integer) Largest amount of memory, in bytes, historically used by the young generation heap. + ========= `survivor`:: -(object) -Contains statistics about memory usage by the survivor space for the node. +(object) Contains statistics about memory usage by the survivor space for the node. + .Properties of `survivor` [%collapsible%open] ========= + `used`:: -(<>) -Memory used by the survivor space. +(<>) Memory used by the survivor space. `used_in_bytes`:: -(integer) -Memory, in bytes, used by the survivor space. +(integer) Memory, in bytes, used by the survivor space. `max`:: -(<>) -Maximum amount of memory available for use by the survivor space. +(<>) Maximum amount of memory available for use by the survivor space. `max_in_bytes`:: -(integer) -Maximum amount of memory, in bytes, available for use by the survivor space. +(integer) Maximum amount of memory, in bytes, available for use by the survivor space. `peak_used`:: -(<>) -Largest amount of memory historically used by the survivor space. +(<>) Largest amount of memory historically used by the survivor space. `peak_used_in_bytes`:: -(integer) -Largest amount of memory, in bytes, historically used by the survivor space. +(integer) Largest amount of memory, in bytes, historically used by the survivor space. `peak_max`:: -(<>) -Largest amount of memory historically used by the survivor space. +(<>) Largest amount of memory historically used by the survivor space. `peak_max_in_bytes`:: -(integer) -Largest amount of memory, in bytes, historically used by the survivor space. +(integer) Largest amount of memory, in bytes, historically used by the survivor space. + ========= `old`:: -(object) -Contains statistics about memory usage by the old generation heap for the node. +(object) Contains statistics about memory usage by the old generation heap for the node. + .Properties of `old` [%collapsible%open] ========= + `used`:: -(<>) -Memory used by the old generation heap. +(<>) Memory used by the old generation heap. `used_in_bytes`:: -(integer) -Memory, in bytes, used by the old generation heap. +(integer) Memory, in bytes, used by the old generation heap. `max`:: -(<>) -Maximum amount of memory available for use by the old generation heap. +(<>) Maximum amount of memory available for use by the old generation heap. `max_in_bytes`:: -(integer) -Maximum amount of memory, in bytes, available for use by the old generation -heap. +(integer) Maximum amount of memory, in bytes, available for use by the old generation heap. `peak_used`:: -(<>) -Largest amount of memory historically used by the old generation heap. +(<>) Largest amount of memory historically used by the old generation heap. `peak_used_in_bytes`:: -(integer) -Largest amount of memory, in bytes, historically used by the old generation -heap. +(integer) Largest amount of memory, in bytes, historically used by the old generation heap. `peak_max`:: -(<>) -Highest memory limit historically available for use by the old generation heap. +(<>) Highest memory limit historically available for use by the old generation heap. `peak_max_in_bytes`:: -(integer) -Highest memory limit, in bytes, historically available for use by the old -generation heap. +(integer) Highest memory limit, in bytes, historically available for use by the old generation heap. + ========= ======== ======= `threads`:: -(object) -Contains statistics about JVM thread usage for the node. +(object) Contains statistics about JVM thread usage for the node. + .Properties of `threads` [%collapsible%open] ======= + `count`:: -(integer) -Number of active threads in use by JVM. +(integer) Number of active threads in use by JVM. `peak_count`:: -(integer) -Highest number of threads used by JVM. +(integer) Highest number of threads used by JVM. + ======= `gc`:: -(object) -Contains statistics about JVM garbage collectors for the node. +(object) Contains statistics about JVM garbage collectors for the node. + .Properties of `gc` [%collapsible%open] ======= + `collectors`:: -(object) -Contains statistics about JVM garbage collectors for the node. +(object) Contains statistics about JVM garbage collectors for the node. + .Properties of `collectors` [%collapsible%open] ======== + `young`:: -(object) -Contains statistics about JVM garbage collectors that collect young generation -objects for the node. +(object) Contains statistics about JVM garbage collectors that collect young generation objects for the node. + .Properties of `young` [%collapsible%open] ========= + `collection_count`:: -(integer) -Number of JVM garbage collectors that collect young generation objects. +(integer) Number of JVM garbage collectors that collect young generation objects. `collection_time`:: -(<>) -Total time spent by JVM collecting young generation objects. +(<>) Total time spent by JVM collecting young generation objects. `collection_time_in_millis`:: -(integer) -Total time in milliseconds spent by JVM collecting young generation objects. +(integer) Total time in milliseconds spent by JVM collecting young generation objects. + ========= `old`:: -(object) -Contains statistics about JVM garbage collectors that collect old generation -objects for the node. +(object) Contains statistics about JVM garbage collectors that collect old generation objects for the node. + .Properties of `old` [%collapsible%open] ========= + `collection_count`:: -(integer) -Number of JVM garbage collectors that collect old generation objects. +(integer) Number of JVM garbage collectors that collect old generation objects. `collection_time`:: -(<>) -Total time spent by JVM collecting old generation objects. +(<>) Total time spent by JVM collecting old generation objects. `collection_time_in_millis`:: -(integer) -Total time in milliseconds spent by JVM collecting old generation objects. +(integer) Total time in milliseconds spent by JVM collecting old generation objects. + ========= ======== ======= `buffer_pools`:: -(object) -Contains statistics about JVM buffer pools for the node. +(object) Contains statistics about JVM buffer pools for the node. + .Properties of `buffer_pools` [%collapsible%open] ======= + `mapped`:: -(object) -Contains statistics about mapped JVM buffer pools for the node. +(object) Contains statistics about mapped JVM buffer pools for the node. + .Properties of `mapped` [%collapsible%open] ======== + `count`:: -(integer) -Number of mapped buffer pools. +(integer) Number of mapped buffer pools. `used`:: -(<>) -Size of mapped buffer pools. +(<>) Size of mapped buffer pools. `used_in_bytes`:: -(integer) -Size, in bytes, of mapped buffer pools. +(integer) Size, in bytes, of mapped buffer pools. `total_capacity`:: -(<>) -Total capacity of mapped buffer pools. +(<>) Total capacity of mapped buffer pools. `total_capacity_in_bytes`:: -(integer) -Total capacity, in bytes, of mapped buffer pools. +(integer) Total capacity, in bytes, of mapped buffer pools. + ======== `direct`:: -(object) -Contains statistics about direct JVM buffer pools for the node. +(object) Contains statistics about direct JVM buffer pools for the node. + .Properties of `direct` [%collapsible%open] ======== + `count`:: -(integer) -Number of direct buffer pools. +(integer) Number of direct buffer pools. `used`:: -(<>) -Size of direct buffer pools. +(<>) Size of direct buffer pools. `used_in_bytes`:: -(integer) -Size, in bytes, of direct buffer pools. +(integer) Size, in bytes, of direct buffer pools. `total_capacity`:: -(<>) -Total capacity of direct buffer pools. +(<>) Total capacity of direct buffer pools. `total_capacity_in_bytes`:: -(integer) -Total capacity, in bytes, of direct buffer pools. +(integer) Total capacity, in bytes, of direct buffer pools. + ======== ======= `classes`:: -(object) -Contains statistics about classes loaded by JVM for the node. +(object) Contains statistics about classes loaded by JVM for the node. + .Properties of `classes` [%collapsible%open] ======= + `current_loaded_count`:: -(integer) -Number of classes currently loaded by JVM. +(integer) Number of classes currently loaded by JVM. `total_loaded_count`:: -(integer) -Total number of classes loaded since the JVM started. +(integer) Total number of classes loaded since the JVM started. `total_unloaded_count`:: -(integer) -Total number of classes unloaded since the JVM started. +(integer) Total number of classes unloaded since the JVM started. + ======= ====== [[cluster-nodes-stats-api-response-body-repositories]] `repositories`:: -(object) -Statistics about snapshot repositories. +(object) Statistics about snapshot repositories. + .Properties of `repositories` [%collapsible%open] ====== + ``:: -(object) -Contains repository throttling statistics for the node. +(object) Contains repository throttling statistics for the node. + .Properties of `` [%collapsible%open] ======= + `total_read_throttled_time_nanos`:: -(integer) -Total number of nanos which node had to wait during recovery. +(integer) Total number of nanos which node had to wait during recovery. `total_write_throttled_time_nanos`:: -(integer) -Total number of nanos which node had to wait during snapshotting. +(integer) Total number of nanos which node had to wait during snapshotting. + ======= ====== [[cluster-nodes-stats-api-response-body-threadpool]] `thread_pool`:: -(object) -Contains thread pool statistics for the node +(object) Contains thread pool statistics for the node + .Properties of `thread_pool` [%collapsible%open] ====== + ``:: -(object) -Contains statistics about the thread pool for the node. +(object) Contains statistics about the thread pool for the node. + .Properties of `` [%collapsible%open] ======= + `threads`:: -(integer) -Number of threads in the thread pool. +(integer) Number of threads in the thread pool. `queue`:: -(integer) -Number of tasks in queue for the thread pool. +(integer) Number of tasks in queue for the thread pool. `active`:: -(integer) -Number of active threads in the thread pool. +(integer) Number of active threads in the thread pool. `rejected`:: -(integer) -Number of tasks rejected by the thread pool executor. +(integer) Number of tasks rejected by the thread pool executor. `largest`:: -(integer) -Highest number of active threads in the thread pool. +(integer) Highest number of active threads in the thread pool. `completed`:: -(integer) -Number of tasks completed by the thread pool executor. +(integer) Number of tasks completed by the thread pool executor. + ======= ====== [[cluster-nodes-stats-api-response-body-fs]] `fs`:: -(object) -Contains file store statistics for the node. +(object) Contains file store statistics for the node. + .Properties of `fs` [%collapsible%open] ====== + `timestamp`:: -(integer) -Last time the file stores statistics were refreshed. Recorded in -milliseconds since the {wikipedia}/Unix_time[Unix Epoch]. +(integer) Last time the file stores statistics were refreshed. +Recorded in milliseconds since the {wikipedia}/Unix_time[Unix Epoch]. `total`:: -(object) -Contains statistics for all file stores of the node. +(object) Contains statistics for all file stores of the node. + .Properties of `total` [%collapsible%open] ======= + `total`:: -(<>) -Total size of all file stores. +(<>) Total size of all file stores. `total_in_bytes`:: -(integer) -Total size (in bytes) of all file stores. +(integer) Total size (in bytes) of all file stores. `free`:: -(<>) -Total unallocated disk space in all file stores. +(<>) Total unallocated disk space in all file stores. `free_in_bytes`:: -(integer) -Total number of unallocated bytes in all file stores. +(integer) Total number of unallocated bytes in all file stores. `available`:: -(<>) -Total disk space available to this Java virtual machine on all file -stores. Depending on OS or process level restrictions (e.g. XFS quotas), this might appear -less than `free`. This is the actual amount of free disk -space the {es} node can utilise. +(<>) Total disk space available to this Java virtual machine on all file stores. +Depending on OS or process level restrictions (e.g. XFS quotas), this might appear less than `free`. +This is the actual amount of free disk space the {es} node can utilise. `available_in_bytes`:: -(integer) -Total number of bytes available to this Java virtual machine on all file -stores. Depending on OS or process level restrictions (e.g. XFS quotas), this might appear -less than `free_in_bytes`. This is the actual amount of free disk -space the {es} node can utilise. +(integer) Total number of bytes available to this Java virtual machine on all file stores. +Depending on OS or process level restrictions (e.g. XFS quotas), this might appear less than `free_in_bytes`. +This is the actual amount of free disk space the {es} node can utilise. + ======= [[cluster-nodes-stats-fs-data]] `data`:: -(array of objects) -List of all file stores. +(array of objects) List of all file stores. + .Properties of `data` [%collapsible%open] ======= + `path`:: -(string) -Path to the file store. +(string) Path to the file store. `mount`:: -(string) -Mount point of the file store (ex: /dev/sda2). +(string) Mount point of the file store (ex: /dev/sda2). `type`:: -(string) -Type of the file store (ex: ext4). +(string) Type of the file store (ex: ext4). `total`:: -(<>) -Total size of the file store. +(<>) Total size of the file store. `total_in_bytes`:: -(integer) -Total size (in bytes) of the file store. +(integer) Total size (in bytes) of the file store. `free`:: -(<>) -Total amount of unallocated disk space in the file store. +(<>) Total amount of unallocated disk space in the file store. `free_in_bytes`:: -(integer) -Total number of unallocated bytes in the file store. +(integer) Total number of unallocated bytes in the file store. `available`:: -(<>) -Total amount of disk space available to this Java virtual machine on this file -store. +(<>) Total amount of disk space available to this Java virtual machine on this file store. `available_in_bytes`:: -(integer) -Total number of bytes available to this Java virtual machine on this file -store. +(integer) Total number of bytes available to this Java virtual machine on this file store. `low_watermark_free_space`:: -(<>) -The effective low disk watermark for this data path on this node: when a node -has less free space than this value for at least one data path, its disk usage -has exceeded the low watermark. See <> for more -information about disk watermarks and their effects on shard allocation. +(<>) The effective low disk watermark for this data path on this node: when a node has less free space than this value for at least one data path, its disk usage has exceeded the low watermark. +See <> for more information about disk watermarks and their effects on shard allocation. `low_watermark_free_space_in_bytes`:: -(integer) -The effective low disk watermark, in bytes, for this data path on this node: -when a node has less free space than this value for at least one data path, its -disk usage has exceeded the low watermark. See <> -for more information about disk watermarks and their effects on shard -allocation. +(integer) The effective low disk watermark, in bytes, for this data path on this node: +when a node has less free space than this value for at least one data path, its disk usage has exceeded the low watermark. +See <> +for more information about disk watermarks and their effects on shard allocation. `high_watermark_free_space`:: -(<>) -The effective high disk watermark for this data path on this node: when a node -has less free space than this value for at least one data path, its disk usage -has exceeded the high watermark. See <> for more -information about disk watermarks and their effects on shard allocation. +(<>) The effective high disk watermark for this data path on this node: when a node has less free space than this value for at least one data path, its disk usage has exceeded the high watermark. +See <> for more information about disk watermarks and their effects on shard allocation. `high_watermark_free_space_in_bytes`:: -(integer) -The effective high disk watermark, in bytes, for this data path on this node: -when a node has less free space than this value for at least one data path, its -disk usage has exceeded the high watermark. See <> -for more information about disk watermarks and their effects on shard -allocation. +(integer) The effective high disk watermark, in bytes, for this data path on this node: +when a node has less free space than this value for at least one data path, its disk usage has exceeded the high watermark. +See <> +for more information about disk watermarks and their effects on shard allocation. `flood_stage_free_space`:: -(<>) -The effective flood stage disk watermark for this data path on this node: when -a node has less free space than this value for at least one data path, its disk -usage has exceeded the flood stage watermark. See -<> for more information about disk watermarks and -their effects on shard allocation. +(<>) The effective flood stage disk watermark for this data path on this node: when a node has less free space than this value for at least one data path, its disk usage has exceeded the flood stage watermark. +See +<> for more information about disk watermarks and their effects on shard allocation. `flood_stage_free_space_in_bytes`:: -(integer) -The effective flood stage disk watermark, in bytes, for this data path on this -node: when a node has less free space than this value for at least one data -path, its disk usage has exceeded the flood stage watermark. See -<> for more information about disk watermarks and -their effects on shard allocation. +(integer) The effective flood stage disk watermark, in bytes, for this data path on this node: when a node has less free space than this value for at least one data path, its disk usage has exceeded the flood stage watermark. +See +<> for more information about disk watermarks and their effects on shard allocation. `frozen_flood_stage_free_space`:: -(<>) -The effective flood stage disk watermark for this data path on a dedicated -frozen node: when a dedicated frozen node has less free space than this value -for at least one data path, its disk usage has exceeded the flood stage -watermark. See <> for more information about disk -watermarks and their effects on shard allocation. +(<>) The effective flood stage disk watermark for this data path on a dedicated frozen node: when a dedicated frozen node has less free space than this value for at least one data path, its disk usage has exceeded the flood stage watermark. +See <> for more information about disk watermarks and their effects on shard allocation. `frozen_flood_stage_free_space_in_bytes`:: -(integer) -The effective flood stage disk watermark, in bytes, for this data path on a -dedicated frozen node: when a dedicated frozen node has less free space than -this value for at least one data path, its disk usage has exceeded the flood -stage watermark. See <> for more information about -disk watermarks and their effects on shard allocation. +(integer) The effective flood stage disk watermark, in bytes, for this data path on a dedicated frozen node: when a dedicated frozen node has less free space than this value for at least one data path, its disk usage has exceeded the flood stage watermark. +See <> for more information about disk watermarks and their effects on shard allocation. + ======= `io_stats` (Linux only):: -(objects) -Contains I/O statistics for the node. +(objects) Contains I/O statistics for the node. + .Properties of `io_stats` [%collapsible%open] ======= + `devices` (Linux only):: -(array) -Array of disk metrics for each device that is backing an {es} data path. -These disk metrics are probed periodically and averages between the last -probe and the current probe are computed. +(array) Array of disk metrics for each device that is backing an {es} data path. +These disk metrics are probed periodically and averages between the last probe and the current probe are computed. + .Properties of `devices` [%collapsible%open] ======== + `device_name` (Linux only):: -(string) -The Linux device name. +(string) The Linux device name. `operations` (Linux only):: -(integer) -The total number of read and write operations for the device completed since -starting {es}. +(integer) The total number of read and write operations for the device completed since starting {es}. `read_operations` (Linux only):: -(integer) -The total number of read operations for the device completed since starting +(integer) The total number of read operations for the device completed since starting {es}. `write_operations` (Linux only):: -(integer) -The total number of write operations for the device completed since starting +(integer) The total number of write operations for the device completed since starting {es}. `read_kilobytes` (Linux only):: -(integer) -The total number of kilobytes read for the device since starting {es}. +(integer) The total number of kilobytes read for the device since starting {es}. `write_kilobytes` (Linux only):: -(integer) -The total number of kilobytes written for the device since starting {es}. +(integer) The total number of kilobytes written for the device since starting {es}. `io_time_in_millis` (Linux only):: -(integer) -The total time in milliseconds spent performing I/O operations for the device -since starting {es}. +(integer) The total time in milliseconds spent performing I/O operations for the device since starting {es}. + ======== `total` (Linux only):: -(object) -The sum of the disk metrics for all devices that back an {es} data path. +(object) The sum of the disk metrics for all devices that back an {es} data path. + .Properties of `total` [%collapsible%open] ======== + `operations` (Linux only):: - (integer) - The total number of read and write operations across all devices used by - {es} completed since starting {es}. +(integer) The total number of read and write operations across all devices used by +{es} completed since starting {es}. `read_operations` (Linux only):: - (integer) - The total number of read operations for across all devices used by {es} - completed since starting {es}. +(integer) The total number of read operations for across all devices used by {es} +completed since starting {es}. `write_operations` (Linux only):: - (integer) - The total number of write operations across all devices used by {es} - completed since starting {es}. +(integer) The total number of write operations across all devices used by {es} +completed since starting {es}. `read_kilobytes` (Linux only):: - (integer) - The total number of kilobytes read across all devices used by {es} since - starting {es}. +(integer) The total number of kilobytes read across all devices used by {es} since starting {es}. `write_kilobytes` (Linux only):: - (integer) - The total number of kilobytes written across all devices used by {es} since - starting {es}. +(integer) The total number of kilobytes written across all devices used by {es} since starting {es}. `io_time_in_millis` (Linux only):: - (integer) - The total time in milliseconds spent performing I/O operations across all - devices used by {es} since starting {es}. +(integer) The total time in milliseconds spent performing I/O operations across all devices used by {es} since starting {es}. + ======== ======= @@ -2009,176 +1645,136 @@ The sum of the disk metrics for all devices that back an {es} data path. [[cluster-nodes-stats-api-response-body-transport]] `transport`:: -(object) -Contains transport statistics for the node. +(object) Contains transport statistics for the node. + .Properties of `transport` [%collapsible%open] ====== + `server_open`:: -(integer) -Current number of inbound TCP connections used for internal communication between nodes. +(integer) Current number of inbound TCP connections used for internal communication between nodes. `total_outbound_connections`:: -(integer) -The cumulative number of outbound transport connections that this node has -opened since it started. Each transport connection may comprise multiple TCP -connections but is only counted once in this statistic. Transport connections -are typically <> so this statistic should -remain constant in a stable cluster. +(integer) The cumulative number of outbound transport connections that this node has opened since it started. +Each transport connection may comprise multiple TCP connections but is only counted once in this statistic. +Transport connections are typically <> so this statistic should remain constant in a stable cluster. `rx_count`:: -(integer) -Total number of RX (receive) packets received by the node during internal -cluster communication. +(integer) Total number of RX (receive) packets received by the node during internal cluster communication. `rx_size`:: -(<>) -Size of RX packets received by the node during internal cluster communication. +(<>) Size of RX packets received by the node during internal cluster communication. `rx_size_in_bytes`:: -(integer) -Size, in bytes, of RX packets received by the node during internal cluster -communication. +(integer) Size, in bytes, of RX packets received by the node during internal cluster communication. `tx_count`:: -(integer) -Total number of TX (transmit) packets sent by the node during internal cluster -communication. +(integer) Total number of TX (transmit) packets sent by the node during internal cluster communication. `tx_size`:: -(<>) -Size of TX packets sent by the node during internal cluster communication. +(<>) Size of TX packets sent by the node during internal cluster communication. `tx_size_in_bytes`:: -(integer) -Size, in bytes, of TX packets sent by the node during internal cluster -communication. +(integer) Size, in bytes, of TX packets sent by the node during internal cluster communication. `inbound_handling_time_histogram`:: -(array) -The distribution of the time spent handling each inbound message on a transport -thread, represented as a histogram. +(array) The distribution of the time spent handling each inbound message on a transport thread, represented as a histogram. + .Properties of `inbound_handling_time_histogram` [%collapsible] ======= + `ge`:: -(string) -The inclusive lower bound of the bucket as a human-readable string. May be -omitted on the first bucket if this bucket has no lower bound. +(string) The inclusive lower bound of the bucket as a human-readable string. +May be omitted on the first bucket if this bucket has no lower bound. `ge_millis`:: -(integer) -The inclusive lower bound of the bucket in milliseconds. May be omitted on the -first bucket if this bucket has no lower bound. +(integer) The inclusive lower bound of the bucket in milliseconds. +May be omitted on the first bucket if this bucket has no lower bound. `lt`:: -(string) -The exclusive upper bound of the bucket as a human-readable string. May be -omitted on the last bucket if this bucket has no upper bound. +(string) The exclusive upper bound of the bucket as a human-readable string. +May be omitted on the last bucket if this bucket has no upper bound. `lt_millis`:: -(integer) -The exclusive upper bound of the bucket in milliseconds. May be omitted on the -last bucket if this bucket has no upper bound. +(integer) The exclusive upper bound of the bucket in milliseconds. +May be omitted on the last bucket if this bucket has no upper bound. `count`:: -(integer) -The number of times a transport thread took a period of time within the bounds -of this bucket to handle an inbound message. +(integer) The number of times a transport thread took a period of time within the bounds of this bucket to handle an inbound message. + ======= `outbound_handling_time_histogram`:: -(array) -The distribution of the time spent sending each outbound transport message on a -transport thread, represented as a histogram. +(array) The distribution of the time spent sending each outbound transport message on a transport thread, represented as a histogram. + .Properties of `outbound_handling_time_histogram` [%collapsible] ======= + `ge`:: -(string) -The inclusive lower bound of the bucket as a human-readable string. May be -omitted on the first bucket if this bucket has no lower bound. +(string) The inclusive lower bound of the bucket as a human-readable string. +May be omitted on the first bucket if this bucket has no lower bound. `ge_millis`:: -(integer) -The inclusive lower bound of the bucket in milliseconds. May be omitted on the -first bucket if this bucket has no lower bound. +(integer) The inclusive lower bound of the bucket in milliseconds. +May be omitted on the first bucket if this bucket has no lower bound. `lt`:: -(string) -The exclusive upper bound of the bucket as a human-readable string. May be -omitted on the last bucket if this bucket has no upper bound. +(string) The exclusive upper bound of the bucket as a human-readable string. +May be omitted on the last bucket if this bucket has no upper bound. `lt_millis`:: -(integer) -The exclusive upper bound of the bucket in milliseconds. May be omitted on the -last bucket if this bucket has no upper bound. +(integer) The exclusive upper bound of the bucket in milliseconds. +May be omitted on the last bucket if this bucket has no upper bound. `count`:: -(integer) -The number of times a transport thread took a period of time within the bounds -of this bucket to send a transport message. +(integer) The number of times a transport thread took a period of time within the bounds of this bucket to send a transport message. + ======= `actions`:: -(object) -An action-by-action breakdown of the transport traffic handled by this node, -showing the total amount of traffic and a histogram of message sizes for -incoming requests and outgoing responses. +(object) An action-by-action breakdown of the transport traffic handled by this node, showing the total amount of traffic and a histogram of message sizes for incoming requests and outgoing responses. + .Properties of `actions.*.requests` and `actions.*.responses` [%collapsible] ======= + `count`:: -(integer) -The total number of requests received, or responses sent, for the current -action. +(integer) The total number of requests received, or responses sent, for the current action. `total_size`:: -(<>) -The total size (as a human-readable string) of all requests received, or -responses sent, for the current action. +(<>) The total size (as a human-readable string) of all requests received, or responses sent, for the current action. `total_size_in_bytes`:: -(integer) -The total size in bytes of all requests received, or responses sent, for the -current action. +(integer) The total size in bytes of all requests received, or responses sent, for the current action. `histogram`:: -(array) -A breakdown of the distribution of sizes of requests received, or responses -sent, for the current action. +(array) A breakdown of the distribution of sizes of requests received, or responses sent, for the current action. + .Properties of `histogram` [%collapsible] ======== + `ge`:: -(<>) -The inclusive lower bound of the bucket as a human-readable string. May be -omitted on the first bucket if this bucket has no lower bound. +(<>) The inclusive lower bound of the bucket as a human-readable string. +May be omitted on the first bucket if this bucket has no lower bound. `ge_bytes`:: -(integer) -The inclusive lower bound of the bucket in bytes. May be omitted on the first -bucket if this bucket has no lower bound. +(integer) The inclusive lower bound of the bucket in bytes. +May be omitted on the first bucket if this bucket has no lower bound. `lt`:: -(<>) -The exclusive upper bound of the bucket as a human-readable string. May be -omitted on the last bucket if this bucket has no upper bound. +(<>) The exclusive upper bound of the bucket as a human-readable string. +May be omitted on the last bucket if this bucket has no upper bound. `lt_bytes`:: -(integer) -The exclusive upper bound of the bucket in bytes. May be omitted on the last -bucket if this bucket has no upper bound. +(integer) The exclusive upper bound of the bucket in bytes. +May be omitted on the last bucket if this bucket has no upper bound. `count`:: -(integer) -The number of times a request was received, or a response sent, with a size -within the bounds of this bucket. +(integer) The number of times a request was received, or a response sent, with a size within the bounds of this bucket. + ======== ======= @@ -2186,389 +1782,297 @@ within the bounds of this bucket. [[cluster-nodes-stats-api-response-body-http]] `http`:: -(object) -Contains http statistics for the node. +(object) Contains http statistics for the node. + .Properties of `http` [%collapsible%open] ====== + `current_open`:: -(integer) -Current number of open HTTP connections for the node. +(integer) Current number of open HTTP connections for the node. `total_opened`:: -(integer) -Total number of HTTP connections opened for the node. +(integer) Total number of HTTP connections opened for the node. `clients`:: -(array of objects) -Information on current and recently-closed HTTP client connections. +(array of objects) Information on current and recently-closed HTTP client connections. Clients that have been closed longer than the <> setting will not be represented here. + .Properties of `clients` [%collapsible%open] ======= + `id`:: -(integer) -Unique ID for the HTTP client. +(integer) Unique ID for the HTTP client. `agent`:: -(string) -Reported agent for the HTTP client. If unavailable, this property is not -included in the response. +(string) Reported agent for the HTTP client. +If unavailable, this property is not included in the response. `local_address`:: -(string) -Local address for the HTTP connection. +(string) Local address for the HTTP connection. `remote_address`:: -(string) -Remote address for the HTTP connection. +(string) Remote address for the HTTP connection. `last_uri`:: -(string) -The URI of the client's most recent request. +(string) The URI of the client's most recent request. `x_forwarded_for`:: -(string) -Value from the client's `x-forwarded-for` HTTP header. If unavailable, this -property is not included in the response. +(string) Value from the client's `x-forwarded-for` HTTP header. +If unavailable, this property is not included in the response. `x_opaque_id`:: -(string) -Value from the client's `x-opaque-id` HTTP header. If unavailable, this property -is not included in the response. +(string) Value from the client's `x-opaque-id` HTTP header. +If unavailable, this property is not included in the response. `opened_time_millis`:: -(integer) -Time at which the client opened the connection. +(integer) Time at which the client opened the connection. `closed_time_millis`:: -(integer) -Time at which the client closed the connection if the connection is closed. +(integer) Time at which the client closed the connection if the connection is closed. `last_request_time_millis`:: -(integer) -Time of the most recent request from this client. +(integer) Time of the most recent request from this client. `request_count`:: -(integer) -Number of requests from this client. +(integer) Number of requests from this client. `request_size_bytes`:: -(integer) -Cumulative size in bytes of all requests from this client. +(integer) Cumulative size in bytes of all requests from this client. + ======= ====== [[cluster-nodes-stats-api-response-body-breakers]] `breakers`:: -(object) -Contains circuit breaker statistics for the node. +(object) Contains circuit breaker statistics for the node. + .Properties of `breakers` [%collapsible%open] ====== + ``:: -(object) -Contains statistics for the circuit breaker. +(object) Contains statistics for the circuit breaker. + .Properties of `` [%collapsible%open] ======= + `limit_size_in_bytes`:: -(integer) -Memory limit, in bytes, for the circuit breaker. +(integer) Memory limit, in bytes, for the circuit breaker. `limit_size`:: -(<>) -Memory limit for the circuit breaker. +(<>) Memory limit for the circuit breaker. `estimated_size_in_bytes`:: -(integer) -Estimated memory used, in bytes, for the operation. +(integer) Estimated memory used, in bytes, for the operation. `estimated_size`:: -(<>) -Estimated memory used for the operation. +(<>) Estimated memory used for the operation. `overhead`:: -(float) -A constant that all estimates for the circuit breaker are multiplied with to -calculate a final estimate. +(float) A constant that all estimates for the circuit breaker are multiplied with to calculate a final estimate. `tripped`:: -(integer) -Total number of times the circuit breaker has been triggered and prevented an -out of memory error. +(integer) Total number of times the circuit breaker has been triggered and prevented an out of memory error. + ======= ====== [[cluster-nodes-stats-api-response-body-script]] `script`:: -(object) -Contains script statistics for the node. +(object) Contains script statistics for the node. + .Properties of `script` [%collapsible%open] ====== + `compilations`:: -(integer) -Total number of inline script compilations performed by the node. +(integer) Total number of inline script compilations performed by the node. `compilations_history`:: -(object) -Contains this recent history of script compilations +(object) Contains this recent history of script compilations .Properties of `compilations_history` [%collapsible%open] ======= + `5m`:: -(long) -The number of script compilations in the last five minutes. +(long) The number of script compilations in the last five minutes. `15m`:: -(long) -The number of script compilations in the last fifteen minutes. +(long) The number of script compilations in the last fifteen minutes. `24h`:: -(long) -The number of script compilations in the last twenty-four hours. +(long) The number of script compilations in the last twenty-four hours. + ======= `cache_evictions`:: -(integer) -Total number of times the script cache has evicted old data. +(integer) Total number of times the script cache has evicted old data. `cache_evictions_history`:: -(object) -Contains this recent history of script cache evictions +(object) Contains this recent history of script cache evictions .Properties of `cache_evictions` [%collapsible%open] ======= `5m`:: -(long) -The number of script cache evictions in the last five minutes. +(long) The number of script cache evictions in the last five minutes. `15m`:: -(long) -The number of script cache evictions in the last fifteen minutes. +(long) The number of script cache evictions in the last fifteen minutes. `24h`:: -(long) -The number of script cache evictions in the last twenty-four hours. +(long) The number of script cache evictions in the last twenty-four hours. ======= `compilation_limit_triggered`:: -(integer) -Total number of times the <> circuit breaker has limited inline script compilations. +(integer) Total number of times the <> circuit breaker has limited inline script compilations. + ====== [[cluster-nodes-stats-api-response-body-discovery]] `discovery`:: -(object) -Contains node discovery statistics for the node. +(object) Contains node discovery statistics for the node. + .Properties of `discovery` [%collapsible%open] ====== + `cluster_state_queue`:: -(object) -Contains statistics for the cluster state queue of the node. +(object) Contains statistics for the cluster state queue of the node. + .Properties of `cluster_state_queue` [%collapsible%open] ======= `total`:: -(integer) -Total number of cluster states in queue. +(integer) Total number of cluster states in queue. `pending`:: -(integer) -Number of pending cluster states in queue. +(integer) Number of pending cluster states in queue. `committed`:: -(integer) -Number of committed cluster states in queue. +(integer) Number of committed cluster states in queue. + ======= `published_cluster_states`:: -(object) -Contains statistics for the published cluster states of the node. +(object) Contains statistics for the published cluster states of the node. + .Properties of `published_cluster_states` [%collapsible%open] ======= + `full_states`:: -(integer) -Number of published cluster states. +(integer) Number of published cluster states. `incompatible_diffs`:: -(integer) -Number of incompatible differences between published cluster states. +(integer) Number of incompatible differences between published cluster states. `compatible_diffs`:: -(integer) -Number of compatible differences between published cluster states. +(integer) Number of compatible differences between published cluster states. + ======= `cluster_state_update`:: -(object) -Contains low-level statistics about how long various activities took during -cluster state updates while the node was the elected master. Omitted if the -node is not master-eligible. Every field whose name ends in `_time` within this -object is also represented as a raw number of milliseconds in a field whose -name ends in `_time_millis`. The human-readable fields with a `_time` suffix -are only returned if requested with the `?human=true` query parameter. +(object) Contains low-level statistics about how long various activities took during cluster state updates while the node was the elected master. +Omitted if the node is not master-eligible. +Every field whose name ends in `_time` within this object is also represented as a raw number of milliseconds in a field whose name ends in `_time_millis`. +The human-readable fields with a `_time` suffix are only returned if requested with the `?human=true` query parameter. + .Properties of `cluster_state_update` [%collapsible] ======= + `unchanged`:: -(object) -Contains statistics about cluster state update attempts that did not change the -cluster state. +(object) Contains statistics about cluster state update attempts that did not change the cluster state. + .Properties of `unchanged` [%collapsible] ======== + `count`:: -(long) -The number of cluster state update attempts that did not change the cluster -state since the node started. +(long) The number of cluster state update attempts that did not change the cluster state since the node started. `computation_time`:: -(<>) -The cumulative amount of time spent computing no-op cluster state updates since -the node started. +(<>) The cumulative amount of time spent computing no-op cluster state updates since the node started. `notification_time`:: -(<>) -The cumulative amount of time spent notifying listeners of a no-op cluster -state update since the node started. +(<>) The cumulative amount of time spent notifying listeners of a no-op cluster state update since the node started. ======== `success`:: -(object) -Contains statistics about cluster state update attempts that successfully -changed the cluster state. +(object) Contains statistics about cluster state update attempts that successfully changed the cluster state. + .Properties of `success` [%collapsible] ======== + `count`:: -(long) -The number of cluster state update attempts that successfully changed the -cluster state since the node started. +(long) The number of cluster state update attempts that successfully changed the cluster state since the node started. `computation_time`:: -(<>) -The cumulative amount of time spent computing cluster state updates that were -ultimately successful since the node started. +(<>) The cumulative amount of time spent computing cluster state updates that were ultimately successful since the node started. `publication_time`:: -(<>) -The cumulative amount of time spent publishing cluster state updates which -ultimately succeeded, which includes everything from the start of the -publication (i.e. just after the computation of the new cluster state) until -the publication has finished and the master node is ready to start processing -the next state update. This includes the time measured by +(<>) The cumulative amount of time spent publishing cluster state updates which ultimately succeeded, which includes everything from the start of the publication (i.e. just after the computation of the new cluster state) until the publication has finished and the master node is ready to start processing the next state update. +This includes the time measured by `context_construction_time`, `commit_time`, `completion_time` and `master_apply_time`. `context_construction_time`:: -(<>) -The cumulative amount of time spent constructing a _publication context_ since -the node started for publications that ultimately succeeded. This statistic -includes the time spent computing the difference between the current and new -cluster state preparing a serialized representation of this difference. +(<>) The cumulative amount of time spent constructing a _publication context_ since the node started for publications that ultimately succeeded. +This statistic includes the time spent computing the difference between the current and new cluster state preparing a serialized representation of this difference. `commit_time`:: -(<>) -The cumulative amount of time spent waiting for a successful cluster state -update to _commit_, which measures the time from the start of each publication -until a majority of the master-eligible nodes have written the state to disk -and confirmed the write to the elected master. +(<>) The cumulative amount of time spent waiting for a successful cluster state update to _commit_, which measures the time from the start of each publication until a majority of the master-eligible nodes have written the state to disk and confirmed the write to the elected master. `completion_time`:: -(<>) -The cumulative amount of time spent waiting for a successful cluster state -update to _complete_, which measures the time from the start of each -publication until all the other nodes have notified the elected master that -they have applied the cluster state. +(<>) The cumulative amount of time spent waiting for a successful cluster state update to _complete_, which measures the time from the start of each publication until all the other nodes have notified the elected master that they have applied the cluster state. `master_apply_time`:: -(<>) -The cumulative amount of time spent successfully applying cluster state updates -on the elected master since the node started. +(<>) The cumulative amount of time spent successfully applying cluster state updates on the elected master since the node started. `notification_time`:: -(<>) -The cumulative amount of time spent notifying listeners of a successful cluster -state update since the node started. +(<>) The cumulative amount of time spent notifying listeners of a successful cluster state update since the node started. ======== `failure`:: -(object) -Contains statistics about cluster state update attempts that did not -successfully change the cluster state, typically because a new master node was -elected before completion. +(object) Contains statistics about cluster state update attempts that did not successfully change the cluster state, typically because a new master node was elected before completion. + .Properties of `failure` [%collapsible] ======== + `count`:: -(long) -The number of cluster state update attempts that failed to change the cluster -state since the node started. +(long) The number of cluster state update attempts that failed to change the cluster state since the node started. `computation_time`:: -(<>) -The cumulative amount of time spent computing cluster state updates that were -ultimately unsuccessful since the node started. +(<>) The cumulative amount of time spent computing cluster state updates that were ultimately unsuccessful since the node started. `publication_time`:: -(<>) -The cumulative amount of time spent publishing cluster state updates which -ultimately failed, which includes everything from the start of the -publication (i.e. just after the computation of the new cluster state) until -the publication has finished and the master node is ready to start processing -the next state update. This includes the time measured by +(<>) The cumulative amount of time spent publishing cluster state updates which ultimately failed, which includes everything from the start of the publication (i.e. just after the computation of the new cluster state) until the publication has finished and the master node is ready to start processing the next state update. +This includes the time measured by `context_construction_time`, `commit_time`, `completion_time` and `master_apply_time`. `context_construction_time`:: -(<>) -The cumulative amount of time spent constructing a _publication context_ since -the node started for publications that ultimately failed. This statistic -includes the time spent computing the difference between the current and new -cluster state preparing a serialized representation of this difference. +(<>) The cumulative amount of time spent constructing a _publication context_ since the node started for publications that ultimately failed. +This statistic includes the time spent computing the difference between the current and new cluster state preparing a serialized representation of this difference. `commit_time`:: -(<>) -The cumulative amount of time spent waiting for an unsuccessful cluster state -update to _commit_, which measures the time from the start of each publication -until a majority of the master-eligible nodes have written the state to disk -and confirmed the write to the elected master. +(<>) The cumulative amount of time spent waiting for an unsuccessful cluster state update to _commit_, which measures the time from the start of each publication until a majority of the master-eligible nodes have written the state to disk and confirmed the write to the elected master. `completion_time`:: -(<>) -The cumulative amount of time spent waiting for an unsuccessful cluster state -update to _complete_, which measures the time from the start of each -publication until all the other nodes have notified the elected master that -they have applied the cluster state. +(<>) The cumulative amount of time spent waiting for an unsuccessful cluster state update to _complete_, which measures the time from the start of each publication until all the other nodes have notified the elected master that they have applied the cluster state. `master_apply_time`:: -(<>) -The cumulative amount of time spent unsuccessfully applying cluster state -updates on the elected master since the node started. +(<>) The cumulative amount of time spent unsuccessfully applying cluster state updates on the elected master since the node started. `notification_time`:: -(<>) -The cumulative amount of time spent notifying listeners of a failed cluster -state update since the node started. +(<>) The cumulative amount of time spent notifying listeners of a failed cluster state update since the node started. ======== ======= @@ -2576,72 +2080,61 @@ state update since the node started. [[cluster-nodes-stats-api-response-body-ingest]] `ingest`:: -(object) -Contains ingest statistics for the node. +(object) Contains ingest statistics for the node. + .Properties of `ingest` [%collapsible%open] ====== + `total`:: -(object) -Contains statistics about ingest operations for the node. +(object) Contains statistics about ingest operations for the node. + .Properties of `total` [%collapsible%open] ======= + `count`:: -(integer) -Total number of documents ingested during the lifetime of this node. +(integer) Total number of documents ingested during the lifetime of this node. `time`:: -(<>) -Total time spent preprocessing ingest documents during the lifetime of this -node. +(<>) Total time spent preprocessing ingest documents during the lifetime of this node. `time_in_millis`:: -(integer) -Total time, in milliseconds, spent preprocessing ingest documents during the -lifetime of this node. +(integer) Total time, in milliseconds, spent preprocessing ingest documents during the lifetime of this node. `current`:: -(integer) -Total number of documents currently being ingested. +(integer) Total number of documents currently being ingested. `failed`:: -(integer) -Total number of failed ingest operations during the lifetime of this node. +(integer) Total number of failed ingest operations during the lifetime of this node. + ======= `pipelines`:: -(object) -Contains statistics about ingest pipelines for the node. +(object) Contains statistics about ingest pipelines for the node. + .Properties of `pipelines` [%collapsible%open] ======= + ``:: -(object) -Contains statistics about the ingest pipeline. +(object) Contains statistics about the ingest pipeline. + .Properties of `` [%collapsible%open] ======== + `count`:: -(integer) -Number of documents preprocessed by the ingest pipeline. +(integer) Number of documents preprocessed by the ingest pipeline. `time`:: -(<>) -Total time spent preprocessing documents in the ingest pipeline. +(<>) Total time spent preprocessing documents in the ingest pipeline. `time_in_millis`:: -(integer) -Total time, in milliseconds, spent preprocessing documents in the ingest -pipeline. +(integer) Total time, in milliseconds, spent preprocessing documents in the ingest pipeline. `failed`:: -(integer) -Total number of failed operations for the ingest pipeline. +(integer) Total number of failed operations for the ingest pipeline. `ingested_as_first_pipeline`:: (<>) @@ -2672,38 +2165,33 @@ run after a reroute processor, or is within a pipeline processor. Instead, the document size is added to the stat value of the pipeline which initially ingested the document. `processors`:: -(array of objects) -Contains statistics for the ingest processors for the ingest pipeline. +(array of objects) Contains statistics for the ingest processors for the ingest pipeline. + .Properties of `processors` [%collapsible%open] ========= + ``:: -(object) -Contains statistics for the ingest processor. +(object) Contains statistics for the ingest processor. + .Properties of `` [%collapsible%open] ========== `count`:: -(integer) -Number of documents transformed by the processor. +(integer) Number of documents transformed by the processor. `time`:: -(<>) -Time spent by the processor transforming documents. +(<>) Time spent by the processor transforming documents. `time_in_millis`:: -(integer) -Time, in milliseconds, spent by the processor transforming documents. +(integer) Time, in milliseconds, spent by the processor transforming documents. `current`:: -(integer) -Number of documents currently being transformed by the processor. +(integer) Number of documents currently being transformed by the processor. `failed`:: -(integer) -Number of failed operations for the processor. +(integer) Number of failed operations for the processor. + ========== ========= ======== @@ -2712,227 +2200,179 @@ Number of failed operations for the processor. [[cluster-nodes-stats-api-response-body-indexing-pressure]] `indexing_pressure`:: -(object) -Contains <> statistics for the node. +(object) Contains <> statistics for the node. + .Properties of `indexing_pressure` [%collapsible%open] ====== + `memory`:: -(object) -Contains statistics for memory consumption from indexing load. +(object) Contains statistics for memory consumption from indexing load. + .Properties of `` [%collapsible%open] ======= + `current`:: -(object) -Contains statistics for current indexing load. +(object) Contains statistics for current indexing load. + .Properties of `` [%collapsible%open] ======== + `combined_coordinating_and_primary`:: -(<>) -Memory consumed by indexing requests in the coordinating or primary stage. This -value is not the sum of coordinating and primary as a node can reuse the -coordinating memory if the primary stage is executed locally. +(<>) Memory consumed by indexing requests in the coordinating or primary stage. +This value is not the sum of coordinating and primary as a node can reuse the coordinating memory if the primary stage is executed locally. `combined_coordinating_and_primary_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the coordinating or primary -stage. This value is not the sum of coordinating and primary as a node can -reuse the coordinating memory if the primary stage is executed locally. +(integer) Memory consumed, in bytes, by indexing requests in the coordinating or primary stage. +This value is not the sum of coordinating and primary as a node can reuse the coordinating memory if the primary stage is executed locally. `coordinating`:: -(<>) -Memory consumed by indexing requests in the coordinating stage. +(<>) Memory consumed by indexing requests in the coordinating stage. `coordinating_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the coordinating stage. +(integer) Memory consumed, in bytes, by indexing requests in the coordinating stage. `primary`:: -(<>) -Memory consumed by indexing requests in the primary stage. +(<>) Memory consumed by indexing requests in the primary stage. `primary_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the primary stage. +(integer) Memory consumed, in bytes, by indexing requests in the primary stage. `replica`:: -(<>) -Memory consumed by indexing requests in the replica stage. +(<>) Memory consumed by indexing requests in the replica stage. `replica_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the replica stage. +(integer) Memory consumed, in bytes, by indexing requests in the replica stage. `all`:: -(<>) -Memory consumed by indexing requests in the coordinating, primary, or replica stage. +(<>) Memory consumed by indexing requests in the coordinating, primary, or replica stage. `all_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the coordinating, primary, -or replica stage. +(integer) Memory consumed, in bytes, by indexing requests in the coordinating, primary, or replica stage. + ======== `total`:: -(object) -Contains statistics for the cumulative indexing load since the node started. +(object) Contains statistics for the cumulative indexing load since the node started. + .Properties of `` [%collapsible%open] ======== + `combined_coordinating_and_primary`:: -(<>) -Memory consumed by indexing requests in the coordinating or primary stage. This -value is not the sum of coordinating and primary as a node can reuse the -coordinating memory if the primary stage is executed locally. +(<>) Memory consumed by indexing requests in the coordinating or primary stage. +This value is not the sum of coordinating and primary as a node can reuse the coordinating memory if the primary stage is executed locally. `combined_coordinating_and_primary_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the coordinating or primary -stage. This value is not the sum of coordinating and primary as a node can -reuse the coordinating memory if the primary stage is executed locally. +(integer) Memory consumed, in bytes, by indexing requests in the coordinating or primary stage. +This value is not the sum of coordinating and primary as a node can reuse the coordinating memory if the primary stage is executed locally. `coordinating`:: -(<>) -Memory consumed by indexing requests in the coordinating stage. +(<>) Memory consumed by indexing requests in the coordinating stage. `coordinating_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the coordinating stage. +(integer) Memory consumed, in bytes, by indexing requests in the coordinating stage. `primary`:: -(<>) -Memory consumed by indexing requests in the primary stage. +(<>) Memory consumed by indexing requests in the primary stage. `primary_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the primary stage. +(integer) Memory consumed, in bytes, by indexing requests in the primary stage. `replica`:: -(<>) -Memory consumed by indexing requests in the replica stage. +(<>) Memory consumed by indexing requests in the replica stage. `replica_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the replica stage. +(integer) Memory consumed, in bytes, by indexing requests in the replica stage. `all`:: -(<>) -Memory consumed by indexing requests in the coordinating, primary, or replica stage. +(<>) Memory consumed by indexing requests in the coordinating, primary, or replica stage. `all_in_bytes`:: -(integer) -Memory consumed, in bytes, by indexing requests in the coordinating, primary, -or replica stage. +(integer) Memory consumed, in bytes, by indexing requests in the coordinating, primary, or replica stage. `coordinating_rejections`:: -(integer) -Number of indexing requests rejected in the coordinating stage. +(integer) Number of indexing requests rejected in the coordinating stage. `primary_rejections`:: -(integer) -Number of indexing requests rejected in the primary stage. +(integer) Number of indexing requests rejected in the primary stage. `replica_rejections`:: -(integer) -Number of indexing requests rejected in the replica stage. +(integer) Number of indexing requests rejected in the replica stage. + ======== `limit`:: -(<>) -Configured memory limit for the indexing requests. Replica requests have an -automatic limit that is 1.5x this value. +(<>) Configured memory limit for the indexing requests. +Replica requests have an automatic limit that is 1.5x this value. `limit_in_bytes`:: -(integer) -Configured memory limit, in bytes, for the indexing requests. Replica requests -have an automatic limit that is 1.5x this value. +(integer) Configured memory limit, in bytes, for the indexing requests. +Replica requests have an automatic limit that is 1.5x this value. + ======= ====== [[cluster-nodes-stats-api-response-body-adaptive-selection]] `adaptive_selection`:: -(object) -Contains adaptive selection statistics for the node. +(object) Contains adaptive selection statistics for the node. + .Properties of `adaptive_selection` [%collapsible%open] ====== + `outgoing_searches`:: -(integer) -The number of outstanding search requests from the node these stats are for -to the keyed node. +(integer) The number of outstanding search requests from the node these stats are for to the keyed node. `avg_queue_size`:: -(integer) -The exponentially weighted moving average queue size of search requests on -the keyed node. +(integer) The exponentially weighted moving average queue size of search requests on the keyed node. `avg_service_time`:: -(<>) -The exponentially weighted moving average service time of search requests on -the keyed node. +(<>) The exponentially weighted moving average service time of search requests on the keyed node. `avg_service_time_ns`:: -(integer) -The exponentially weighted moving average service time, in nanoseconds, of -search requests on the keyed node. +(integer) The exponentially weighted moving average service time, in nanoseconds, of search requests on the keyed node. `avg_response_time`:: -(<>) -The exponentially weighted moving average response time of search requests -on the keyed node. +(<>) The exponentially weighted moving average response time of search requests on the keyed node. `avg_response_time_ns`:: -(integer) -The exponentially weighted moving average response time, in nanoseconds, of -search requests on the keyed node. +(integer) The exponentially weighted moving average response time, in nanoseconds, of search requests on the keyed node. `rank`:: -(string) -The rank of this node; used for shard selection when routing search -requests. +(string) The rank of this node; used for shard selection when routing search requests. + ====== [[cluster-nodes-stats-api-response-body-allocations]] `allocations`:: -(object) -Contains allocations statistics for the node. +(object) Contains allocations statistics for the node. + .Properties of `allocations` [%collapsible%open] ====== + `shards`:: -(integer) -The number of shards currently allocated to this node +(integer) The number of shards currently allocated to this node `undesired_shards`:: -(integer) -The amount of shards that are scheduled to be moved elsewhere in the cluster -if desired balance allocator is used or -1 if any other allocator is used. +(integer) The amount of shards that are scheduled to be moved elsewhere in the cluster if desired balance allocator is used or -1 if any other allocator is used. `forecasted_ingest_load`:: -(double) -Total forecasted ingest load of all shards assigned to this node +(double) Total forecasted ingest load of all shards assigned to this node `forecasted_disk_usage`:: -(<>) -Forecasted size of all shards assigned to the node +(<>) Forecasted size of all shards assigned to the node `forecasted_disk_usage_bytes`:: -(integer) -Forecasted size, in bytes, of all shards assigned to the node +(integer) Forecasted size, in bytes, of all shards assigned to the node `current_disk_usage`:: -(<>) -Current size of all shards assigned to the node +(<>) Current size of all shards assigned to the node `current_disk_usage_bytes`:: -(integer) -Current size, in bytes, of all shards assigned to the node +(integer) Current size, in bytes, of all shards assigned to the node + ====== ===== ==== @@ -2973,8 +2413,7 @@ GET /_nodes/stats/indices/fielddata?level=shards&fields=field1,field2 GET /_nodes/stats/indices/fielddata?fields=field* ---- -You can get statistics about search groups for searches executed -on this node. +You can get statistics about search groups for searches executed on this node. [source,console,id=nodes-stats-groups] ---- @@ -2988,8 +2427,7 @@ GET /_nodes/stats/indices?groups=foo,bar [[cluster-nodes-stats-ingest-ex]] ===== Retrieve ingest statistics only -To return only ingest-related node statistics, set the `` path -parameter to `ingest` and use the +To return only ingest-related node statistics, set the `` path parameter to `ingest` and use the <> query parameter. [source,console,id=nodes-stats-filter-path] @@ -2997,8 +2435,7 @@ parameter to `ingest` and use the GET /_nodes/stats/ingest?filter_path=nodes.*.ingest ---- -You can use the `metric` and `filter_path` query parameters to get the same -response. +You can use the `metric` and `filter_path` query parameters to get the same response. [source,console,id=nodes-stats-metric-filter-path] ---- diff --git a/docs/reference/cluster/stats.asciidoc b/docs/reference/cluster/stats.asciidoc index 26b3553c3c17f..3b429ef427071 100644 --- a/docs/reference/cluster/stats.asciidoc +++ b/docs/reference/cluster/stats.asciidoc @@ -1,12 +1,12 @@ [[cluster-stats]] === Cluster stats API + ++++ Cluster stats ++++ Returns cluster statistics. - [[cluster-stats-api-request]] ==== {api-request-title} @@ -23,67 +23,55 @@ Returns cluster statistics. [[cluster-stats-api-desc]] ==== {api-description-title} -The Cluster Stats API allows to retrieve statistics from a cluster wide -perspective. The API returns basic index metrics (shard numbers, store size, -memory usage) and information about the current nodes that form the cluster -(number, roles, os, jvm versions, memory usage, cpu and installed plugins). - +The Cluster Stats API allows to retrieve statistics from a cluster wide perspective. +The API returns basic index metrics (shard numbers, store size, memory usage) and information about the current nodes that form the cluster (number, roles, os, jvm versions, memory usage, cpu and installed plugins). [[cluster-stats-api-path-params]] ==== {api-path-parms-title} - include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=node-filter] - [[cluster-stats-api-query-params]] ==== {api-query-parms-title} `timeout`:: -(Optional, <>) -Period to wait for each node to respond. If a node does not respond before its -timeout expires, the response does not include its stats. However, timed out -nodes are included in the response's `_nodes.failed` property. Defaults to no -timeout. +(Optional, <>) Period to wait for each node to respond. +If a node does not respond before its timeout expires, the response does not include its stats. +However, timed out nodes are included in the response's `_nodes.failed` property. +Defaults to no timeout. [role="child_attributes"] [[cluster-stats-api-response-body]] ==== {api-response-body-title} `_nodes`:: -(object) -Contains statistics about the number of nodes selected by the request's +(object) Contains statistics about the number of nodes selected by the request's <>. + .Properties of `_nodes` [%collapsible%open] ==== `total`:: -(integer) -Total number of nodes selected by the request. +(integer) Total number of nodes selected by the request. `successful`:: -(integer) -Number of nodes that responded successfully to the request. +(integer) Number of nodes that responded successfully to the request. `failed`:: -(integer) -Number of nodes that rejected the request or failed to respond. If this value -is not `0`, a reason for the rejection or failure is included in the response. +(integer) Number of nodes that rejected the request or failed to respond. +If this value is not `0`, a reason for the rejection or failure is included in the response. + ==== `cluster_name`:: -(string) -Name of the cluster, based on the <> setting. +(string) Name of the cluster, based on the <> setting. `cluster_uuid`:: -(string) -Unique identifier for the cluster. +(string) Unique identifier for the cluster. `timestamp`:: (integer) -{wikipedia}/Unix_time[Unix timestamp], in milliseconds, of -the last time the cluster statistics were refreshed. +{wikipedia}/Unix_time[Unix timestamp], in milliseconds, of the last time the cluster statistics were refreshed. `status`:: include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=cluster-health-status] @@ -92,771 +80,602 @@ See <>. [[cluster-stats-api-response-body-indices]] `indices`:: -(object) -Contains statistics about indices with shards assigned to selected nodes. +(object) Contains statistics about indices with shards assigned to selected nodes. + .Properties of `indices` [%collapsible%open] ==== + `count`:: -(integer) -Total number of indices with shards assigned to selected nodes. +(integer) Total number of indices with shards assigned to selected nodes. `shards`:: -(object) -Contains statistics about shards assigned to selected nodes. +(object) Contains statistics about shards assigned to selected nodes. + .Properties of `shards` [%collapsible%open] ===== `total`:: -(integer) -Total number of shards assigned to selected nodes. +(integer) Total number of shards assigned to selected nodes. `primaries`:: -(integer) -Number of primary shards assigned to selected nodes. +(integer) Number of primary shards assigned to selected nodes. `replication`:: -(float) -Ratio of replica shards to primary shards across all selected nodes. +(float) Ratio of replica shards to primary shards across all selected nodes. `index`:: -(object) -Contains statistics about shards assigned to selected nodes. +(object) Contains statistics about shards assigned to selected nodes. + .Properties of `index` [%collapsible%open] ====== `shards`:: -(object) -Contains statistics about the number of shards assigned to selected nodes. +(object) Contains statistics about the number of shards assigned to selected nodes. + .Properties of `shards` [%collapsible%open] ======= `min`:: -(integer) -Minimum number of shards in an index, counting only shards assigned to -selected nodes. +(integer) Minimum number of shards in an index, counting only shards assigned to selected nodes. `max`:: -(integer) -Maximum number of shards in an index, counting only shards assigned to -selected nodes. +(integer) Maximum number of shards in an index, counting only shards assigned to selected nodes. `avg`:: -(float) -Mean number of shards in an index, counting only shards assigned to -selected nodes. +(float) Mean number of shards in an index, counting only shards assigned to selected nodes. + ======= `primaries`:: -(object) -Contains statistics about the number of primary shards assigned to selected -nodes. +(object) Contains statistics about the number of primary shards assigned to selected nodes. + .Properties of `primaries` [%collapsible%open] ======= + `min`:: -(integer) -Minimum number of primary shards in an index, counting only shards assigned -to selected nodes. +(integer) Minimum number of primary shards in an index, counting only shards assigned to selected nodes. `max`:: -(integer) -Maximum number of primary shards in an index, counting only shards assigned -to selected nodes. +(integer) Maximum number of primary shards in an index, counting only shards assigned to selected nodes. `avg`:: -(float) -Mean number of primary shards in an index, counting only shards assigned to -selected nodes. +(float) Mean number of primary shards in an index, counting only shards assigned to selected nodes. + ======= `replication`:: -(object) -Contains statistics about the number of replication shards assigned to selected -nodes. +(object) Contains statistics about the number of replication shards assigned to selected nodes. + .Properties of `replication` [%collapsible%open] ======= + `min`:: -(float) -Minimum replication factor in an index, counting only shards assigned to -selected nodes. +(float) Minimum replication factor in an index, counting only shards assigned to selected nodes. `max`:: -(float) -Maximum replication factor in an index, counting only shards assigned to -selected nodes. +(float) Maximum replication factor in an index, counting only shards assigned to selected nodes. `avg`:: -(float) -Mean replication factor in an index, counting only shards assigned to selected -nodes. +(float) Mean replication factor in an index, counting only shards assigned to selected nodes. + ======= ====== ===== `docs`:: -(object) -Contains counts for documents in selected nodes. +(object) Contains counts for documents in selected nodes. + .Properties of `docs` [%collapsible%open] ===== + `count`:: -(integer) -Total number of non-deleted documents across all primary shards assigned to -selected nodes. +(integer) Total number of non-deleted documents across all primary shards assigned to selected nodes. + -This number is based on documents in Lucene segments and may include documents -from nested fields. +This number is based on documents in Lucene segments and may include documents from nested fields. `deleted`:: -(integer) -Total number of deleted documents across all primary shards assigned to -selected nodes. +(integer) Total number of deleted documents across all primary shards assigned to selected nodes. + -This number is based on documents in Lucene segments. {es} reclaims the disk -space of deleted Lucene documents when a segment is merged. +This number is based on documents in Lucene segments. {es} reclaims the disk space of deleted Lucene documents when a segment is merged. `total_size_in_bytes`:: (integer) Total size in bytes across all primary shards assigned to selected nodes. + +`total_size`:: +(string) +Total size across all primary shards assigned to selected nodes, as a human-readable string. ===== `store`:: -(object) -Contains statistics about the size of shards assigned to selected nodes. +(object) Contains statistics about the size of shards assigned to selected nodes. + .Properties of `store` [%collapsible%open] ===== + `size`:: -(<>) -Total size of all shards assigned to selected nodes. +(<>) Total size of all shards assigned to selected nodes. `size_in_bytes`:: -(integer) -Total size, in bytes, of all shards assigned to selected nodes. +(integer) Total size, in bytes, of all shards assigned to selected nodes. `total_data_set_size`:: -(<>) -Total data set size of all shards assigned to selected nodes. -This includes the size of shards not stored fully on the nodes, such as the -cache for <>. +(<>) Total data set size of all shards assigned to selected nodes. +This includes the size of shards not stored fully on the nodes, such as the cache for <>. `total_data_set_size_in_bytes`:: -(integer) -Total data set size, in bytes, of all shards assigned to selected nodes. -This includes the size of shards not stored fully on the nodes, such as the -cache for <>. +(integer) Total data set size, in bytes, of all shards assigned to selected nodes. +This includes the size of shards not stored fully on the nodes, such as the cache for <>. `reserved`:: -(<>) -A prediction of how much larger the shard stores will eventually grow due to -ongoing peer recoveries, restoring snapshots, and similar activities. +(<>) A prediction of how much larger the shard stores will eventually grow due to ongoing peer recoveries, restoring snapshots, and similar activities. `reserved_in_bytes`:: -(integer) -A prediction, in bytes, of how much larger the shard stores will eventually -grow due to ongoing peer recoveries, restoring snapshots, and similar -activities. +(integer) A prediction, in bytes, of how much larger the shard stores will eventually grow due to ongoing peer recoveries, restoring snapshots, and similar activities. + ===== `fielddata`:: -(object) -Contains statistics about the <> of selected nodes. +(object) Contains statistics about the <> of selected nodes. + .Properties of `fielddata` [%collapsible%open] ===== + `memory_size`:: -(<>) -Total amount of memory used for the field data cache across all shards -assigned to selected nodes. +(<>) Total amount of memory used for the field data cache across all shards assigned to selected nodes. `memory_size_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for the field data cache across all -shards assigned to selected nodes. +(integer) Total amount, in bytes, of memory used for the field data cache across all shards assigned to selected nodes. `evictions`:: -(integer) -Total number of evictions from the field data cache across all shards assigned -to selected nodes. +(integer) Total number of evictions from the field data cache across all shards assigned to selected nodes. `global_ordinals.build_time`:: -(<>) -The total time spent building global ordinals for all fields. +(<>) The total time spent building global ordinals for all fields. `global_ordinals.build_time_in_millis`:: -(integer) -The total time, in milliseconds, spent building global ordinals for all fields. +(integer) The total time, in milliseconds, spent building global ordinals for all fields. `global_ordinals.fields.[field-name].build_time`:: -(<>) -The total time spent building global ordinals for field with specified name. +(<>) The total time spent building global ordinals for field with specified name. `global_ordinals.fields.[field-name].build_time_in_millis`:: -(integer) -The total time, in milliseconds, spent building global ordinals for field with specified name. +(integer) The total time, in milliseconds, spent building global ordinals for field with specified name. `global_ordinals.fields.[field-name].shard_max_value_count`:: -(long) -The total time spent building global ordinals for field with specified name. +(long) The total time spent building global ordinals for field with specified name. + ===== `query_cache`:: -(object) -Contains statistics about the query cache of selected nodes. +(object) Contains statistics about the query cache of selected nodes. + .Properties of `query_cache` [%collapsible%open] ===== + `memory_size`:: -(<>) -Total amount of memory used for the query cache across all shards assigned to -selected nodes. +(<>) Total amount of memory used for the query cache across all shards assigned to selected nodes. `memory_size_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for the query cache across all shards -assigned to selected nodes. +(integer) Total amount, in bytes, of memory used for the query cache across all shards assigned to selected nodes. `total_count`:: -(integer) -Total count of hits and misses in the query cache across all shards assigned to -selected nodes. +(integer) Total count of hits and misses in the query cache across all shards assigned to selected nodes. `hit_count`:: -(integer) -Total count of query cache hits across all shards assigned to selected nodes. +(integer) Total count of query cache hits across all shards assigned to selected nodes. `miss_count`:: -(integer) -Total count of query cache misses across all shards assigned to selected nodes. +(integer) Total count of query cache misses across all shards assigned to selected nodes. `cache_size`:: -(integer) -Total number of entries currently in the query cache across all shards assigned -to selected nodes. +(integer) Total number of entries currently in the query cache across all shards assigned to selected nodes. `cache_count`:: -(integer) -Total number of entries added to the query cache across all shards assigned -to selected nodes. This number includes current and evicted entries. +(integer) Total number of entries added to the query cache across all shards assigned to selected nodes. +This number includes current and evicted entries. `evictions`:: -(integer) -Total number of query cache evictions across all shards assigned to selected -nodes. +(integer) Total number of query cache evictions across all shards assigned to selected nodes. + ===== `completion`:: -(object) -Contains statistics about memory used for completion in selected nodes. +(object) Contains statistics about memory used for completion in selected nodes. + .Properties of `completion` [%collapsible%open] ===== + `size`:: -(<>) -Total amount of memory used for completion across all shards assigned to -selected nodes. +(<>) Total amount of memory used for completion across all shards assigned to selected nodes. `size_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for completion across all shards assigned -to selected nodes. +(integer) Total amount, in bytes, of memory used for completion across all shards assigned to selected nodes. + ===== `segments`:: -(object) -Contains statistics about segments in selected nodes. +(object) Contains statistics about segments in selected nodes. + .Properties of `segments` [%collapsible%open] ===== + `count`:: -(integer) -Total number of segments across all shards assigned to selected nodes. +(integer) Total number of segments across all shards assigned to selected nodes. `memory`:: -(<>) -Total amount of memory used for segments across all shards assigned to selected -nodes. +(<>) Total amount of memory used for segments across all shards assigned to selected nodes. `memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for segments across all shards assigned to -selected nodes. +(integer) Total amount, in bytes, of memory used for segments across all shards assigned to selected nodes. `terms_memory`:: -(<>) -Total amount of memory used for terms across all shards assigned to selected -nodes. +(<>) Total amount of memory used for terms across all shards assigned to selected nodes. `terms_memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for terms across all shards assigned to -selected nodes. +(integer) Total amount, in bytes, of memory used for terms across all shards assigned to selected nodes. `stored_fields_memory`:: -(<>) -Total amount of memory used for stored fields across all shards assigned to -selected nodes. +(<>) Total amount of memory used for stored fields across all shards assigned to selected nodes. `stored_fields_memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for stored fields across all shards -assigned to selected nodes. +(integer) Total amount, in bytes, of memory used for stored fields across all shards assigned to selected nodes. `term_vectors_memory`:: -(<>) -Total amount of memory used for term vectors across all shards assigned to -selected nodes. +(<>) Total amount of memory used for term vectors across all shards assigned to selected nodes. `term_vectors_memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for term vectors across all shards -assigned to selected nodes. +(integer) Total amount, in bytes, of memory used for term vectors across all shards assigned to selected nodes. `norms_memory`:: -(<>) -Total amount of memory used for normalization factors across all shards assigned -to selected nodes. +(<>) Total amount of memory used for normalization factors across all shards assigned to selected nodes. `norms_memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for normalization factors across all -shards assigned to selected nodes. +(integer) Total amount, in bytes, of memory used for normalization factors across all shards assigned to selected nodes. `points_memory`:: -(<>) -Total amount of memory used for points across all shards assigned to selected -nodes. +(<>) Total amount of memory used for points across all shards assigned to selected nodes. `points_memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for points across all shards assigned to -selected nodes. +(integer) Total amount, in bytes, of memory used for points across all shards assigned to selected nodes. `doc_values_memory`:: -(<>) -Total amount of memory used for doc values across all shards assigned to -selected nodes. +(<>) Total amount of memory used for doc values across all shards assigned to selected nodes. `doc_values_memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used for doc values across all shards assigned -to selected nodes. +(integer) Total amount, in bytes, of memory used for doc values across all shards assigned to selected nodes. `index_writer_memory`:: -(<>) -Total amount of memory used by all index writers across all shards assigned to -selected nodes. +(<>) Total amount of memory used by all index writers across all shards assigned to selected nodes. `index_writer_memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used by all index writers across all shards -assigned to selected nodes. +(integer) Total amount, in bytes, of memory used by all index writers across all shards assigned to selected nodes. `version_map_memory`:: -(<>) -Total amount of memory used by all version maps across all shards assigned to -selected nodes. +(<>) Total amount of memory used by all version maps across all shards assigned to selected nodes. `version_map_memory_in_bytes`:: -(integer) -Total amount, in bytes, of memory used by all version maps across all shards -assigned to selected nodes. +(integer) Total amount, in bytes, of memory used by all version maps across all shards assigned to selected nodes. `fixed_bit_set`:: -(<>) -Total amount of memory used by fixed bit sets across all shards assigned to -selected nodes. +(<>) Total amount of memory used by fixed bit sets across all shards assigned to selected nodes. + -Fixed bit sets are used for nested object field types and -type filters for <> fields. +Fixed bit sets are used for nested object field types and type filters for <> fields. `fixed_bit_set_memory_in_bytes`:: -(integer) -Total amount of memory, in bytes, used by fixed bit sets across all shards -assigned to selected nodes. +(integer) Total amount of memory, in bytes, used by fixed bit sets across all shards assigned to selected nodes. `max_unsafe_auto_id_timestamp`:: (integer) -{wikipedia}/Unix_time[Unix timestamp], in milliseconds, of -the most recently retried indexing request. +{wikipedia}/Unix_time[Unix timestamp], in milliseconds, of the most recently retried indexing request. `file_sizes`:: -(object) -This object is not populated by the cluster stats API. +(object) This object is not populated by the cluster stats API. + -To get information on segment files, use the <>. +To get information on segment files, use the <>. + ===== `mappings`:: -(object) -Contains statistics about <> in selected nodes. +(object) Contains statistics about <> in selected nodes. + .Properties of `mappings` [%collapsible%open] ===== + `total_field_count`:: -(integer) -Total number of fields in all non-system indices. +(integer) Total number of fields in all non-system indices. `total_deduplicated_field_count`:: -(integer) -Total number of fields in all non-system indices, accounting for mapping deduplication. +(integer) Total number of fields in all non-system indices, accounting for mapping deduplication. `total_deduplicated_mapping_size`:: -(<>) -Total size of all mappings after deduplication and compression. +(<>) Total size of all mappings after deduplication and compression. `total_deduplicated_mapping_size_in_bytes`:: -(integer) -Total size of all mappings, in bytes, after deduplication and compression. +(integer) Total size of all mappings, in bytes, after deduplication and compression. `field_types`:: -(array of objects) -Contains statistics about <> used in selected -nodes. +(array of objects) Contains statistics about <> used in selected nodes. + .Properties of `field_types` objects [%collapsible%open] ====== + `name`:: -(string) -Field data type used in selected nodes. +(string) Field data type used in selected nodes. `count`:: -(integer) -Number of fields mapped to the field data type in selected nodes. +(integer) Number of fields mapped to the field data type in selected nodes. `index_count`:: -(integer) -Number of indices containing a mapping of the field data type in selected nodes. +(integer) Number of indices containing a mapping of the field data type in selected nodes. `indexed_vector_count`:: -(integer) -For dense_vector field types, number of indexed vector types in selected nodes. +(integer) For dense_vector field types, number of indexed vector types in selected nodes. `indexed_vector_dim_min`:: -(integer) -For dense_vector field types, the minimum dimension of all indexed vector types in selected nodes. +(integer) For dense_vector field types, the minimum dimension of all indexed vector types in selected nodes. `indexed_vector_dim_max`:: -(integer) -For dense_vector field types, the maximum dimension of all indexed vector types in selected nodes. +(integer) For dense_vector field types, the maximum dimension of all indexed vector types in selected nodes. `script_count`:: -(integer) -Number of fields that declare a script. +(integer) Number of fields that declare a script. `lang`:: -(array of strings) -Script languages used for the optional scripts +(array of strings) Script languages used for the optional scripts `lines_max`:: -(integer) -Maximum number of lines for a single field script +(integer) Maximum number of lines for a single field script `lines_total`:: -(integer) -Total number of lines for the scripts +(integer) Total number of lines for the scripts `chars_max`:: -(integer) -Maximum number of characters for a single field script +(integer) Maximum number of characters for a single field script `chars_total`:: -(integer) -Total number of characters for the scripts +(integer) Total number of characters for the scripts `source_max`:: -(integer) -Maximum number of accesses to _source for a single field script +(integer) Maximum number of accesses to _source for a single field script `source_total`:: -(integer) -Total number of accesses to _source for the scripts +(integer) Total number of accesses to _source for the scripts `doc_max`:: -(integer) -Maximum number of accesses to doc_values for a single field script +(integer) Maximum number of accesses to doc_values for a single field script `doc_total`:: -(integer) -Total number of accesses to doc_values for the scripts +(integer) Total number of accesses to doc_values for the scripts + ====== `runtime_field_types`:: -(array of objects) -Contains statistics about <> used in selected -nodes. +(array of objects) Contains statistics about <> used in selected nodes. + .Properties of `runtime_field_types` objects [%collapsible%open] ====== + `name`:: -(string) -Field data type used in selected nodes. +(string) Field data type used in selected nodes. `count`:: -(integer) -Number of runtime fields mapped to the field data type in selected nodes. +(integer) Number of runtime fields mapped to the field data type in selected nodes. `index_count`:: -(integer) -Number of indices containing a mapping of the runtime field data type in selected nodes. +(integer) Number of indices containing a mapping of the runtime field data type in selected nodes. `scriptless_count`:: -(integer) -Number of runtime fields that don't declare a script. +(integer) Number of runtime fields that don't declare a script. `shadowed_count`:: -(integer) -Number of runtime fields that shadow an indexed field. +(integer) Number of runtime fields that shadow an indexed field. `lang`:: -(array of strings) -Script languages used for the runtime fields scripts +(array of strings) Script languages used for the runtime fields scripts `lines_max`:: -(integer) -Maximum number of lines for a single runtime field script +(integer) Maximum number of lines for a single runtime field script `lines_total`:: -(integer) -Total number of lines for the scripts that define the current runtime field data type +(integer) Total number of lines for the scripts that define the current runtime field data type `chars_max`:: -(integer) -Maximum number of characters for a single runtime field script +(integer) Maximum number of characters for a single runtime field script `chars_total`:: -(integer) -Total number of characters for the scripts that define the current runtime field data type +(integer) Total number of characters for the scripts that define the current runtime field data type `source_max`:: -(integer) -Maximum number of accesses to _source for a single runtime field script +(integer) Maximum number of accesses to _source for a single runtime field script `source_total`:: -(integer) -Total number of accesses to _source for the scripts that define the current runtime field data type +(integer) Total number of accesses to _source for the scripts that define the current runtime field data type `doc_max`:: -(integer) -Maximum number of accesses to doc_values for a single runtime field script +(integer) Maximum number of accesses to doc_values for a single runtime field script `doc_total`:: -(integer) -Total number of accesses to doc_values for the scripts that define the current runtime field data type +(integer) Total number of accesses to doc_values for the scripts that define the current runtime field data type ====== ===== `analysis`:: -(object) -Contains statistics about <> +(object) Contains statistics about <> used in selected nodes. + .Properties of `analysis` [%collapsible%open] ===== + `char_filter_types`:: -(array of objects) -Contains statistics about <> types used -in selected nodes. +(array of objects) Contains statistics about <> types used in selected nodes. + .Properties of `char_filter_types` objects [%collapsible%open] ====== + `name`:: -(string) -Character filter type used in selected nodes. +(string) Character filter type used in selected nodes. `count`:: -(integer) -Number of analyzers or normalizers using the character filter type in selected -nodes. +(integer) Number of analyzers or normalizers using the character filter type in selected nodes. `index_count`:: -(integer) -Number of indices the character filter type in selected nodes. +(integer) Number of indices the character filter type in selected nodes. + ====== `tokenizer_types`:: -(array of objects) -Contains statistics about <> types used in -selected nodes. +(array of objects) Contains statistics about <> types used in selected nodes. + .Properties of `tokenizer_types` objects [%collapsible%open] ====== + `name`:: -(string) -Tokenizer type used in selected nodes. +(string) Tokenizer type used in selected nodes. `count`:: -(integer) -Number of analyzers or normalizers using the tokenizer type in selected nodes. +(integer) Number of analyzers or normalizers using the tokenizer type in selected nodes. `index_count`:: -(integer) -Number of indices using the tokenizer type in selected nodes. +(integer) Number of indices using the tokenizer type in selected nodes. + ====== `filter_types`:: -(array of objects) -Contains statistics about <> types used in -selected nodes. +(array of objects) Contains statistics about <> types used in selected nodes. + .Properties of `filter_types` objects [%collapsible%open] ====== + `name`:: -(string) -Token filter type used in selected nodes. +(string) Token filter type used in selected nodes. `count`:: -(integer) -Number of analyzers or normalizers using the token filter type in selected -nodes. +(integer) Number of analyzers or normalizers using the token filter type in selected nodes. `index_count`:: -(integer) -Number of indices using the token filter type in selected nodes. +(integer) Number of indices using the token filter type in selected nodes. + ====== `analyzer_types`:: -(array of objects) -Contains statistics about <> types used in selected -nodes. +(array of objects) Contains statistics about <> types used in selected nodes. + .Properties of `analyzer_types` objects [%collapsible%open] ====== + `name`:: -(string) -Analyzer type used in selected nodes. +(string) Analyzer type used in selected nodes. `count`:: -(integer) -Occurrences of the analyzer type in selected nodes. +(integer) Occurrences of the analyzer type in selected nodes. `index_count`:: -(integer) -Number of indices using the analyzer type in selected nodes. +(integer) Number of indices using the analyzer type in selected nodes. + ====== `built_in_char_filters`:: -(array of objects) -Contains statistics about built-in <> +(array of objects) Contains statistics about built-in <> used in selected nodes. + .Properties of `built_in_char_filters` objects [%collapsible%open] ====== + `name`:: -(string) -Built-in character filter used in selected nodes. +(string) Built-in character filter used in selected nodes. `count`:: -(integer) -Number of analyzers or normalizers using the built-in character filter in -selected nodes. +(integer) Number of analyzers or normalizers using the built-in character filter in selected nodes. `index_count`:: -(integer) -Number of indices using the built-in character filter in selected nodes. +(integer) Number of indices using the built-in character filter in selected nodes. + ====== `built_in_tokenizers`:: -(array of objects) -Contains statistics about built-in <> used in -selected nodes. +(array of objects) Contains statistics about built-in <> used in selected nodes. + .Properties of `built_in_tokenizers` objects [%collapsible%open] ====== + `name`:: -(string) -Built-in tokenizer used in selected nodes. +(string) Built-in tokenizer used in selected nodes. `count`:: -(integer) -Number of analyzers or normalizers using the built-in tokenizer in selected -nodes. +(integer) Number of analyzers or normalizers using the built-in tokenizer in selected nodes. `index_count`:: -(integer) -Number of indices using the built-in tokenizer in selected nodes. +(integer) Number of indices using the built-in tokenizer in selected nodes. + ====== `built_in_filters`:: -(array of objects) -Contains statistics about built-in <> used -in selected nodes. +(array of objects) Contains statistics about built-in <> used in selected nodes. + .Properties of `built_in_filters` objects [%collapsible%open] ====== + `name`:: -(string) -Built-in token filter used in selected nodes. +(string) Built-in token filter used in selected nodes. `count`:: -(integer) -Number of analyzers or normalizers using the built-in token filter in selected -nodes. +(integer) Number of analyzers or normalizers using the built-in token filter in selected nodes. `index_count`:: -(integer) -Number of indices using the built-in token filter in selected nodes. +(integer) Number of indices using the built-in token filter in selected nodes. + ====== `built_in_analyzers`:: -(array of objects) -Contains statistics about built-in <> used in -selected nodes. +(array of objects) Contains statistics about built-in <> used in selected nodes. + .Properties of `built_in_analyzers` objects [%collapsible%open] ====== + `name`:: -(string) -Built-in analyzer used in selected nodes. +(string) Built-in analyzer used in selected nodes. `count`:: -(integer) -Occurrences of the built-in analyzer in selected nodes. +(integer) Occurrences of the built-in analyzer in selected nodes. `index_count`:: -(integer) -Number of indices using the built-in analyzer in selected nodes. +(integer) Number of indices using the built-in analyzer in selected nodes. + ====== `synonyms`:: -(object) -Contains statistics about synonyms defined in <> and <> token filters configuration. +(object) Contains statistics about synonyms defined in <> and <> token filters configuration. + .Properties of `synonyms` objects [%collapsible%open] ====== + `inline`:: -(object) -Inline synonyms defined using `synonyms` configuration in synonym or synonym graph token filters. +(object) Inline synonyms defined using `synonyms` configuration in synonym or synonym graph token filters. + .Properties of `inline` objects @@ -864,430 +683,385 @@ Inline synonyms defined using `synonyms` configuration in synonym or synonym gra ======= `count`:: -(integer) -Occurrences of inline synonyms configuration in selected nodes. +(integer) Occurrences of inline synonyms configuration in selected nodes. Each inline synonyms configuration will be counted separately, regardless of the synonyms defined. Two synonyms configurations with the same synonyms will count as separate ocurrences. `index_count`:: -(integer) -Number of indices that use inline synonyms configuration for synonyms token filters. +(integer) Number of indices that use inline synonyms configuration for synonyms token filters. + ======= `paths`:: -(object) -Contains statistics about synonym files defined as `synonyms_path` in <> and <> token filters configuration. +(object) Contains statistics about synonym files defined as `synonyms_path` in <> and <> token filters configuration. + .Properties of `paths` objects [%collapsible%open] ======= + `count`:: -(integer) -Occurrences of unique synonym paths in selected nodes. +(integer) Occurrences of unique synonym paths in selected nodes. `index_count`:: -(integer) -Number of indices that use `synonyms_path` configuration for synonyms token filters. +(integer) Number of indices that use `synonyms_path` configuration for synonyms token filters. + ======= `sets`:: -(object) -Contains statistics about synonyms sets configured as `synonyms_set` in <> and <> token filters configuration. +(object) Contains statistics about synonyms sets configured as `synonyms_set` in <> and <> token filters configuration. + .Properties of `sets` objects [%collapsible%open] ======= + `count`:: -(integer) -Occurrences of unique synonyms sets in selected nodes. +(integer) Occurrences of unique synonyms sets in selected nodes. `index_count`:: -(integer) -Number of indices that use `synonyms_set` configuration for synonyms token filters. +(integer) Number of indices that use `synonyms_set` configuration for synonyms token filters. + ======= ====== ===== `search`:: -(object) -Contains usage statistics about search requests submitted to selected nodes -that acted as coordinator during the search execution. Search requests are -tracked when they are successfully parsed, regardless of their results: -requests that yield errors after parsing contribute to the usage stats, as -well as requests that don't access any data. +(object) Contains usage statistics about search requests submitted to selected nodes that acted as coordinator during the search execution. +Search requests are tracked when they are successfully parsed, regardless of their results: +requests that yield errors after parsing contribute to the usage stats, as well as requests that don't access any data. + .Properties of `search` objects [%collapsible%open] ===== + `total`:: -(integer) -Total number of incoming search requests. Search requests that don't specify a -request body are not counted. +(integer) Total number of incoming search requests. +Search requests that don't specify a request body are not counted. `queries`:: -(object) -Query types used in selected nodes. For each query, name and number of times -it's been used within the `query` or `post_filter` section is reported. Queries -are counted once per search request, meaning that if the same query type is used -multiple times in the same search request, its counter will be incremented by 1 -rather than by the number of times it's been used in that individual search request. +(object) Query types used in selected nodes. +For each query, name and number of times it's been used within the `query` or `post_filter` section is reported. +Queries are counted once per search request, meaning that if the same query type is used multiple times in the same search request, its counter will be incremented by 1 rather than by the number of times it's been used in that individual search request. `sections`:: -(object) -Search sections used in selected nodes. For each section, name and number of times -it's been used is reported. +(object) Search sections used in selected nodes. +For each section, name and number of times it's been used is reported. ===== `dense_vector`:: -(object) -Contains statistics about indexed dense vector used in selected nodes. +(object) Contains statistics about indexed dense vector used in selected nodes. + .Properties of `dense_vector` [%collapsible%open] ===== + `value_count`:: -(integer) -Total number of dense vector indexed in selected nodes. +(integer) Total number of dense vector indexed in selected nodes. + +===== + +`sparse_vector`:: +(object) Contains statistics about indexed sparse vector used in selected nodes. ++ +.Properties of `sparse_vector` +[%collapsible%open] +===== + +`value_count`:: +(integer) Total number of sparse vectors indexed across all primary shards assigned to selected nodes. + ===== ==== [[cluster-stats-api-response-body-nodes]] `nodes`:: -(object) -Contains statistics about nodes selected by the request's <>. +(object) Contains statistics about nodes selected by the request's <>. + .Properties of `nodes` [%collapsible%open] ==== + `count`:: -(object) -Contains counts for nodes selected by the request's <>. +(object) Contains counts for nodes selected by the request's <>. + .Properties of `count` [%collapsible%open] ===== + `total`:: -(integer) -Total number of selected nodes. +(integer) Total number of selected nodes. `coordinating_only`:: -(integer) -Number of selected nodes without a <>. These nodes are -considered <> nodes. +(integer) Number of selected nodes without a <>. +These nodes are considered <> nodes. ``:: -(integer) -Number of selected nodes with the role. For a list of roles, see +(integer) Number of selected nodes with the role. +For a list of roles, see <>. + ===== `versions`:: -(array of strings) -Array of {es} versions used on selected nodes. +(array of strings) Array of {es} versions used on selected nodes. `os`:: -(object) -Contains statistics about the operating systems used by selected nodes. +(object) Contains statistics about the operating systems used by selected nodes. + .Properties of `os` [%collapsible%open] ===== + `available_processors`:: -(integer) -Number of processors available to JVM across all selected nodes. +(integer) Number of processors available to JVM across all selected nodes. `allocated_processors`:: -(integer) -Number of processors used to calculate thread pool size across all selected -nodes. +(integer) Number of processors used to calculate thread pool size across all selected nodes. + -This number can be set with the `processors` setting of a node and defaults to -the number of processors reported by the OS. In both cases, this number will -never be larger than `32`. +This number can be set with the `processors` setting of a node and defaults to the number of processors reported by the OS. +In both cases, this number will never be larger than `32`. `names`:: -(array of objects) -Contains statistics about operating systems used by selected nodes. +(array of objects) Contains statistics about operating systems used by selected nodes. + .Properties of `names` [%collapsible%open] ====== + `name`::: -(string) -Name of an operating system used by one or more selected nodes. +(string) Name of an operating system used by one or more selected nodes. `count`::: -(string) -Number of selected nodes using the operating system. +(string) Number of selected nodes using the operating system. + ====== `pretty_names`:: -(array of objects) -Contains statistics about operating systems used by selected nodes. +(array of objects) Contains statistics about operating systems used by selected nodes. + .Properties of `pretty_names` [%collapsible%open] ====== + `pretty_name`::: -(string) -Human-readable name of an operating system used by one or more selected nodes. +(string) Human-readable name of an operating system used by one or more selected nodes. `count`::: -(string) -Number of selected nodes using the operating system. +(string) Number of selected nodes using the operating system. + ====== `architectures`:: -(array of objects) -Contains statistics about processor architectures (for example, x86_64 or -aarch64) used by selected nodes. +(array of objects) Contains statistics about processor architectures (for example, x86_64 or aarch64) used by selected nodes. + .Properties of `architectures` [%collapsible%open] ====== + `arch`::: -(string) -Name of an architecture used by one or more selected nodes. +(string) Name of an architecture used by one or more selected nodes. `count`::: -(string) -Number of selected nodes using the architecture. +(string) Number of selected nodes using the architecture. + ====== `mem`:: -(object) -Contains statistics about memory used by selected nodes. +(object) Contains statistics about memory used by selected nodes. + .Properties of `mem` [%collapsible%open] ====== + `total`:: -(<>) -Total amount of physical memory across all selected nodes. +(<>) Total amount of physical memory across all selected nodes. `total_in_bytes`:: -(integer) -Total amount, in bytes, of physical memory across all selected nodes. +(integer) Total amount, in bytes, of physical memory across all selected nodes. `adjusted_total`:: -(<>) -Total amount of memory across all selected nodes, but using the value specified -using the `es.total_memory_bytes` system property instead of measured total -memory for those nodes where that system property was set. +(<>) Total amount of memory across all selected nodes, but using the value specified using the `es.total_memory_bytes` system property instead of measured total memory for those nodes where that system property was set. `adjusted_total_in_bytes`:: -(integer) -Total amount, in bytes, of memory across all selected nodes, but using the -value specified using the `es.total_memory_bytes` system property instead -of measured total memory for those nodes where that system property was set. +(integer) Total amount, in bytes, of memory across all selected nodes, but using the value specified using the `es.total_memory_bytes` system property instead of measured total memory for those nodes where that system property was set. `free`:: -(<>) -Amount of free physical memory across all selected nodes. +(<>) Amount of free physical memory across all selected nodes. `free_in_bytes`:: -(integer) -Amount, in bytes, of free physical memory across all selected nodes. +(integer) Amount, in bytes, of free physical memory across all selected nodes. `used`:: -(<>) -Amount of physical memory in use across all selected nodes. +(<>) Amount of physical memory in use across all selected nodes. `used_in_bytes`:: -(integer) -Amount, in bytes, of physical memory in use across all selected nodes. +(integer) Amount, in bytes, of physical memory in use across all selected nodes. `free_percent`:: -(integer) -Percentage of free physical memory across all selected nodes. +(integer) Percentage of free physical memory across all selected nodes. `used_percent`:: -(integer) -Percentage of physical memory in use across all selected nodes. +(integer) Percentage of physical memory in use across all selected nodes. + ====== ===== `process`:: -(object) -Contains statistics about processes used by selected nodes. +(object) Contains statistics about processes used by selected nodes. + .Properties of `process` [%collapsible%open] ===== + `cpu`:: -(object) -Contains statistics about CPU used by selected nodes. +(object) Contains statistics about CPU used by selected nodes. + .Properties of `cpu` [%collapsible%open] ====== + `percent`:: -(integer) -Percentage of CPU used across all selected nodes. Returns `-1` if -not supported. +(integer) Percentage of CPU used across all selected nodes. +Returns `-1` if not supported. + ====== `open_file_descriptors`:: -(object) -Contains statistics about open file descriptors in selected nodes. +(object) Contains statistics about open file descriptors in selected nodes. + .Properties of `open_file_descriptors` [%collapsible%open] ====== + `min`:: -(integer) -Minimum number of concurrently open file descriptors across all selected nodes. +(integer) Minimum number of concurrently open file descriptors across all selected nodes. Returns `-1` if not supported. `max`:: -(integer) -Maximum number of concurrently open file descriptors allowed across all selected -nodes. Returns `-1` if not supported. +(integer) Maximum number of concurrently open file descriptors allowed across all selected nodes. +Returns `-1` if not supported. `avg`:: -(integer) -Average number of concurrently open file descriptors. Returns `-1` if not -supported. +(integer) Average number of concurrently open file descriptors. +Returns `-1` if not supported. + ====== ===== `jvm`:: -(object) -Contains statistics about the Java Virtual Machines (JVMs) used by selected -nodes. +(object) Contains statistics about the Java Virtual Machines (JVMs) used by selected nodes. + .Properties of `jvm` [%collapsible%open] ===== + `max_uptime`:: -(<>) -Uptime duration since JVM last started. +(<>) Uptime duration since JVM last started. `max_uptime_in_millis`:: -(integer) -Uptime duration, in milliseconds, since JVM last started. +(integer) Uptime duration, in milliseconds, since JVM last started. `versions`:: -(array of objects) -Contains statistics about the JVM versions used by selected nodes. +(array of objects) Contains statistics about the JVM versions used by selected nodes. + .Properties of `versions` [%collapsible%open] ====== + `version`:: -(string) -Version of JVM used by one or more selected nodes. +(string) Version of JVM used by one or more selected nodes. `vm_name`:: -(string) -Name of the JVM. +(string) Name of the JVM. `vm_version`:: -(string) -Full version number of JVM. +(string) Full version number of JVM. + The full version number includes a plus sign (`+`) followed by the build number. `vm_vendor`:: -(string) -Vendor of the JVM. +(string) Vendor of the JVM. `bundled_jdk`:: -(Boolean) -Always `true`. All distributions come with a bundled Java Development Kit (JDK). +(Boolean) Always `true`. +All distributions come with a bundled Java Development Kit (JDK). `using_bundled_jdk`:: -(Boolean) -If `true`, a bundled JDK is in use by JVM. +(Boolean) If `true`, a bundled JDK is in use by JVM. `count`:: -(integer) -Total number of selected nodes using JVM. +(integer) Total number of selected nodes using JVM. + ====== `mem`:: -(object) -Contains statistics about memory used by selected nodes. +(object) Contains statistics about memory used by selected nodes. + .Properties of `mem` [%collapsible%open] ====== + `heap_used`:: -(<>) -Memory currently in use by the heap across all selected nodes. +(<>) Memory currently in use by the heap across all selected nodes. `heap_used_in_bytes`:: -(integer) -Memory, in bytes, currently in use by the heap across all selected nodes. +(integer) Memory, in bytes, currently in use by the heap across all selected nodes. `heap_max`:: -(<>) -Maximum amount of memory, in bytes, available for use by the heap across all -selected nodes. +(<>) Maximum amount of memory, in bytes, available for use by the heap across all selected nodes. `heap_max_in_bytes`:: -(integer) -Maximum amount of memory, in bytes, available for use by the heap across all -selected nodes. +(integer) Maximum amount of memory, in bytes, available for use by the heap across all selected nodes. + ====== `threads`:: -(integer) -Number of active threads in use by JVM across all selected nodes. +(integer) Number of active threads in use by JVM across all selected nodes. + ===== `fs`:: -(object) -Contains statistics about file stores by selected nodes. +(object) Contains statistics about file stores by selected nodes. + .Properties of `fs` [%collapsible%open] ===== + `total`:: -(<>) -Total size of all file stores across all selected nodes. +(<>) Total size of all file stores across all selected nodes. `total_in_bytes`:: -(integer) -Total size, in bytes, of all file stores across all selected nodes. +(integer) Total size, in bytes, of all file stores across all selected nodes. `free`:: -(<>) -Amount of unallocated disk space in file stores across all selected nodes. +(<>) Amount of unallocated disk space in file stores across all selected nodes. `free_in_bytes`:: -(integer) -Total number of unallocated bytes in file stores across all selected nodes. +(integer) Total number of unallocated bytes in file stores across all selected nodes. `available`:: -(<>) -Total amount of disk space available to JVM in file -stores across all selected nodes. +(<>) Total amount of disk space available to JVM in file stores across all selected nodes. + Depending on OS or process-level restrictions, this amount may be less than -`nodes.fs.free`. This is the actual amount of free disk space the selected {es} +`nodes.fs.free`. +This is the actual amount of free disk space the selected {es} nodes can use. `available_in_bytes`:: -(integer) -Total number of bytes available to JVM in file stores -across all selected nodes. +(integer) Total number of bytes available to JVM in file stores across all selected nodes. + Depending on OS or process-level restrictions, this number may be less than -`nodes.fs.free_in_byes`. This is the actual amount of free disk space the -selected {es} nodes can use. +`nodes.fs.free_in_byes`. +This is the actual amount of free disk space the selected {es} nodes can use. + ===== `plugins`:: -(array of objects) -Contains statistics about installed plugins and modules by selected nodes. +(array of objects) Contains statistics about installed plugins and modules by selected nodes. + If no plugins or modules are installed, this array is empty. + @@ -1296,15 +1070,14 @@ If no plugins or modules are installed, this array is empty. ===== ``:: -(object) -Contains statistics about an installed plugin or module. +(object) Contains statistics about an installed plugin or module. + .Properties of `` [%collapsible%open] ====== + `name`::: -(string) -Name of the {es} plugin. +(string) Name of the {es} plugin. `version`::: (string) @@ -1315,235 +1088,195 @@ Name of the {es} plugin. {es} version for which the plugin was built. `java_version`::: -(string) -Java version for which the plugin was built. +(string) Java version for which the plugin was built. `description`::: -(string) -Short description of the plugin. +(string) Short description of the plugin. `classname`::: -(string) -Class name used as the plugin's entry point. +(string) Class name used as the plugin's entry point. `extended_plugins`::: -(array of strings) -An array of other plugins extended by this plugin through the Java Service -Provider Interface (SPI). +(array of strings) An array of other plugins extended by this plugin through the Java Service Provider Interface (SPI). + If this plugin extends no other plugins, this array is empty. `has_native_controller`::: -(Boolean) -If `true`, the plugin has a native controller process. +(Boolean) If `true`, the plugin has a native controller process. + ====== + ===== `network_types`:: -(object) -Contains statistics about the transport and HTTP networks used by selected -nodes. +(object) Contains statistics about the transport and HTTP networks used by selected nodes. + .Properties of `network_types` [%collapsible%open] ===== + `transport_types`:: -(object) -Contains statistics about the transport network types used by selected nodes. +(object) Contains statistics about the transport network types used by selected nodes. + .Properties of `transport_types` [%collapsible%open] ====== + ``:: -(integer) -Number of selected nodes using the transport type. +(integer) Number of selected nodes using the transport type. + ====== `http_types`:: -(object) -Contains statistics about the HTTP network types used by selected nodes. +(object) Contains statistics about the HTTP network types used by selected nodes. + .Properties of `http_types` [%collapsible%open] ====== + ``:: -(integer) -Number of selected nodes using the HTTP type. +(integer) Number of selected nodes using the HTTP type. + ====== ===== `discovery_types`:: -(object) -Contains statistics about the <> used by selected nodes. +(object) Contains statistics about the <> used by selected nodes. + .Properties of `discovery_types` [%collapsible%open] ===== + ``:: -(integer) -Number of selected nodes using the <> to find other nodes. +(integer) Number of selected nodes using the <> to find other nodes. + ===== `packaging_types`:: -(array of objects) -Contains statistics about {es} distributions installed on selected nodes. +(array of objects) Contains statistics about {es} distributions installed on selected nodes. + .Properties of `packaging_types` [%collapsible%open] ===== + `flavor`::: -(string) -Type of {es} distribution. This is always `default`. +(string) Type of {es} distribution. +This is always `default`. `type`::: -(string) -File type, such as `tar` or `zip`, used for the distribution package. +(string) File type, such as `tar` or `zip`, used for the distribution package. `count`::: -(integer) -Number of selected nodes using the distribution flavor and file type. +(integer) Number of selected nodes using the distribution flavor and file type. + ===== + ==== `snapshots`:: -(object) -Contains statistics about the <> activity in the cluster. +(object) Contains statistics about the <> activity in the cluster. + .Properties of `snapshots` [%collapsible%open] ===== `current_counts`::: -(object) -Contains statistics which report the numbers of various ongoing snapshot activities in the cluster. +(object) Contains statistics which report the numbers of various ongoing snapshot activities in the cluster. + .Properties of `current_counts` [%collapsible%open] ====== + `snapshots`::: -(integer) -The total number of snapshots and clones currently being created by the cluster. +(integer) The total number of snapshots and clones currently being created by the cluster. `shard_snapshots`::: -(integer) -The total number of outstanding shard snapshots in the cluster. +(integer) The total number of outstanding shard snapshots in the cluster. `snapshot_deletions`::: -(integer) -The total number of snapshot deletion operations that the cluster is currently -running. +(integer) The total number of snapshot deletion operations that the cluster is currently running. `concurrent_operations`::: -(integer) -The total number of snapshot operations that the cluster is currently running -concurrently. This is the total of the `snapshots` and `snapshot_deletions` +(integer) The total number of snapshot operations that the cluster is currently running concurrently. +This is the total of the `snapshots` and `snapshot_deletions` entries, and is limited by <>. `cleanups`::: -(integer) -The total number of repository cleanup operations that the cluster is currently -running. These operations do not count towards the total number of concurrent -operations. +(integer) The total number of repository cleanup operations that the cluster is currently running. +These operations do not count towards the total number of concurrent operations. + ====== `repositories`::: -(object) -Contains statistics which report the progress of snapshot activities broken down -by repository. This object contains one entry for each repository registered -with the cluster. +(object) Contains statistics which report the progress of snapshot activities broken down by repository. +This object contains one entry for each repository registered with the cluster. + .Properties of `repositories` [%collapsible%open] ====== `current_counts`::: -(object) -Contains statistics which report the numbers of various ongoing snapshot -activities for this repository. +(object) Contains statistics which report the numbers of various ongoing snapshot activities for this repository. + .Properties of `current_counts` [%collapsible%open] ======= + `snapshots`::: -(integer) -The total number of ongoing snapshots in this repository. +(integer) The total number of ongoing snapshots in this repository. `clones`::: -(integer) -The total number of ongoing snapshot clones in this repository. +(integer) The total number of ongoing snapshot clones in this repository. `finalizations`::: -(integer) -The total number of this repository's ongoing snapshots and clone operations -which are mostly complete except for their last "finalization" step. +(integer) The total number of this repository's ongoing snapshots and clone operations which are mostly complete except for their last "finalization" step. `deletions`::: -(integer) -The total number of ongoing snapshot deletion operations in this repository. +(integer) The total number of ongoing snapshot deletion operations in this repository. `snapshot_deletions`::: -(integer) -The total number of snapshots that are currently being deleted from this -repository. +(integer) The total number of snapshots that are currently being deleted from this repository. `active_deletions`::: -(integer) -The total number of ongoing snapshot deletion operations which are currently -active in this repository. Snapshot deletions do not run concurrently with other -snapshot operations, so this may be `0` if any pending deletes are waiting for -other operations to finish. +(integer) The total number of ongoing snapshot deletion operations which are currently active in this repository. +Snapshot deletions do not run concurrently with other snapshot operations, so this may be `0` if any pending deletes are waiting for other operations to finish. `shards`::: -(object) -Contains statistics which report the shard-level progress of ongoing snapshot -activities for a repository. Note that these statistics relate only to ongoing -snapshots. +(object) Contains statistics which report the shard-level progress of ongoing snapshot activities for a repository. +Note that these statistics relate only to ongoing snapshots. + .Properties of `shards` [%collapsible%open] ======== `total`::: -(integer) -The total number of shard snapshots currently tracked by this repository. This -statistic only counts shards in ongoing snapshots, so it will drop when a -snapshot completes and will be `0` if there are no ongoing snapshots. +(integer) The total number of shard snapshots currently tracked by this repository. +This statistic only counts shards in ongoing snapshots, so it will drop when a snapshot completes and will be `0` if there are no ongoing snapshots. `complete`::: -(integer) -The total number of tracked shard snapshots which have completed in this -repository. This statistic only counts shards in ongoing snapshots, so it will -drop when a snapshot completes and will be `0` if there are no ongoing -snapshots. +(integer) The total number of tracked shard snapshots which have completed in this repository. +This statistic only counts shards in ongoing snapshots, so it will drop when a snapshot completes and will be `0` if there are no ongoing snapshots. `incomplete`::: -(integer) -The total number of tracked shard snapshots which have not completed in this -repository. This is the difference between the `total` and `complete` values. +(integer) The total number of tracked shard snapshots which have not completed in this repository. +This is the difference between the `total` and `complete` values. `states`::: -(object) -The total number of shard snapshots in each of the named states in this -repository. These states are an implementation detail of the snapshotting -process which may change between versions. They are included here for expert -users, but should otherwise be ignored. +(object) The total number of shard snapshots in each of the named states in this repository. +These states are an implementation detail of the snapshotting process which may change between versions. +They are included here for expert users, but should otherwise be ignored. ======== ======= `oldest_start_time`::: -(string) -The start time of the oldest running snapshot in this repository. +(string) The start time of the oldest running snapshot in this repository. `oldest_start_time_in_millis`::: -(integer) -The start time of the oldest running snapshot in this repository, represented as -milliseconds since the Unix epoch. +(integer) The start time of the oldest running snapshot in this repository, represented as milliseconds since the Unix epoch. ====== @@ -1599,6 +1332,7 @@ The API returns the following response: "docs": { "count": 10, "deleted": 0, + "total_size": "8.6kb", "total_size_in_bytes": 8833 }, "store": { @@ -1690,6 +1424,9 @@ The API returns the following response: ], "dense_vector": { "value_count": 0 + }, + "sparse_vector": { + "value_count": 0 } }, "nodes": { @@ -1871,8 +1608,7 @@ The API returns the following response: // the response are ignored. So we're really only asserting things about the // the shape of this response, not the values in it. -This API can be restricted to a subset of the nodes using <>: +This API can be restricted to a subset of the nodes using <>: [source,console] -------------------------------------------------- diff --git a/docs/reference/data-streams/change-mappings-and-settings.asciidoc b/docs/reference/data-streams/change-mappings-and-settings.asciidoc index c96f0c7342a96..076b315558b60 100644 --- a/docs/reference/data-streams/change-mappings-and-settings.asciidoc +++ b/docs/reference/data-streams/change-mappings-and-settings.asciidoc @@ -602,7 +602,7 @@ stream's oldest backing index. // TESTRESPONSE[s/"index_uuid": "_eEfRrFHS9OyhqWntkgHAQ"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] // TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.07-000001"/"index_name": $body.data_streams.0.indices.0.index_name/] // TESTRESPONSE[s/"index_name": ".ds-my-data-stream-2099.03.08-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] -// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/] +// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] <1> First item in the `indices` array for `my-data-stream`. This item contains information about the stream's oldest backing index, diff --git a/docs/reference/data-streams/downsampling-manual.asciidoc b/docs/reference/data-streams/downsampling-manual.asciidoc index 8f6b39d2aa0dd..771a08d97d949 100644 --- a/docs/reference/data-streams/downsampling-manual.asciidoc +++ b/docs/reference/data-streams/downsampling-manual.asciidoc @@ -389,7 +389,7 @@ This returns: // TESTRESPONSE[s/"ltOJGmqgTVm4T-Buoe7Acg"/$body.data_streams.0.indices.0.index_uuid/] // TESTRESPONSE[s/"2023-07-26T09:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.start/] // TESTRESPONSE[s/"2023-07-26T13:26:42.000Z"/$body.data_streams.0.time_series.temporal_ranges.0.end/] -// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/] +// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] <1> The backing index for this data stream. Before a backing index can be downsampled, the TSDS needs to be rolled over and diff --git a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc index b89f55dd41575..5b2e2a1ec70a2 100644 --- a/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc +++ b/docs/reference/data-streams/lifecycle/tutorial-migrate-data-stream-from-ilm-to-dsl.asciidoc @@ -147,7 +147,7 @@ and that the next generation index will also be managed by {ilm-init}: // TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] // TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] // TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] -// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/] +// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] <1> The name of the backing index. <2> For each backing index we display the value of the <> @@ -284,7 +284,7 @@ GET _data_stream/dsl-data-stream // TESTRESPONSE[s/"index_uuid": "xCEhwsp8Tey0-FLNFYVwSg"/"index_uuid": $body.data_streams.0.indices.0.index_uuid/] // TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000002"/"index_name": $body.data_streams.0.indices.1.index_name/] // TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] -// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/] +// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] <1> The existing backing index will continue to be managed by {ilm-init} <2> The existing backing index will continue to be managed by {ilm-init} @@ -364,7 +364,7 @@ GET _data_stream/dsl-data-stream // TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] // TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000003"/"index_name": $body.data_streams.0.indices.2.index_name/] // TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8abcd1"/"index_uuid": $body.data_streams.0.indices.2.index_uuid/] -// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/] +// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] <1> The backing indices that existed before rollover will continue to be managed by {ilm-init} <2> The backing indices that existed before rollover will continue to be managed by {ilm-init} @@ -462,7 +462,7 @@ GET _data_stream/dsl-data-stream // TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8DJ5gw"/"index_uuid": $body.data_streams.0.indices.1.index_uuid/] // TESTRESPONSE[s/"index_name": ".ds-dsl-data-stream-2023.10.19-000003"/"index_name": $body.data_streams.0.indices.2.index_name/] // TESTRESPONSE[s/"index_uuid": "PA_JquKGSiKcAKBA8abcd1"/"index_uuid": $body.data_streams.0.indices.2.index_uuid/] -// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/] +// TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW","failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] <1> The write index is now managed by {ilm-init} <2> The `lifecycle` configured on the data stream is now disabled. <3> The next write index will be managed by {ilm-init} diff --git a/docs/reference/datatiers.asciidoc b/docs/reference/datatiers.asciidoc index 4aff273588926..0981e80804383 100644 --- a/docs/reference/datatiers.asciidoc +++ b/docs/reference/datatiers.asciidoc @@ -22,6 +22,9 @@ mounted indices>> of <> exclusively. This extends the storage capacity even further — by up to 20 times compared to the warm tier. +TIP: The performance of an {es} node is often limited by the performance of the underlying storage. +Review our recommendations for optimizing your storage for <> and <>. + IMPORTANT: {es} generally expects nodes within a data tier to share the same hardware profile. Variations not following this recommendation should be carefully architected to avoid <>. diff --git a/docs/reference/docs/bulk.asciidoc b/docs/reference/docs/bulk.asciidoc index 02f7d7e941fe8..69bf3d1b7db5a 100644 --- a/docs/reference/docs/bulk.asciidoc +++ b/docs/reference/docs/bulk.asciidoc @@ -140,7 +140,7 @@ Perl:: Python:: - See https://elasticsearch-py.readthedocs.org/en/master/helpers.html[elasticsearch.helpers.*] + See https://elasticsearch-py.readthedocs.io/en/latest/helpers.html[elasticsearch.helpers.*] JavaScript:: diff --git a/docs/reference/esql/functions/binary.asciidoc b/docs/reference/esql/functions/binary.asciidoc index 431efab1c924a..959bbe11c040e 100644 --- a/docs/reference/esql/functions/binary.asciidoc +++ b/docs/reference/esql/functions/binary.asciidoc @@ -65,6 +65,9 @@ include::types/mul.asciidoc[] [.text-center] image::esql/functions/signature/div.svg[Embedded,opts=inline] +NOTE: Division of two integer types will yield an integer result, rounding towards 0. + If you need floating point division, <> one of the arguments to a `DOUBLE`. + include::types/div.asciidoc[] ==== Modulus `%` diff --git a/docs/reference/esql/functions/kibana/inline_cast.json b/docs/reference/esql/functions/kibana/inline_cast.json new file mode 100644 index 0000000000000..f71572d3d651c --- /dev/null +++ b/docs/reference/esql/functions/kibana/inline_cast.json @@ -0,0 +1,19 @@ +{ + "bool" : "to_boolean", + "boolean" : "to_boolean", + "cartesian_point" : "to_cartesianpoint", + "cartesian_shape" : "to_cartesianshape", + "datetime" : "to_datetime", + "double" : "to_double", + "geo_point" : "to_geopoint", + "geo_shape" : "to_geoshape", + "int" : "to_integer", + "integer" : "to_integer", + "ip" : "to_ip", + "keyword" : "to_string", + "long" : "to_long", + "string" : "to_string", + "text" : "to_string", + "unsigned_long" : "to_unsigned_long", + "version" : "to_version" +} \ No newline at end of file diff --git a/docs/reference/esql/functions/mv-functions.asciidoc b/docs/reference/esql/functions/mv-functions.asciidoc index 1820ea7051f69..0f4f6233d446c 100644 --- a/docs/reference/esql/functions/mv-functions.asciidoc +++ b/docs/reference/esql/functions/mv-functions.asciidoc @@ -8,6 +8,7 @@ {esql} supports these multivalue functions: // tag::mv_list[] +* <> * <> * <> * <> @@ -23,6 +24,7 @@ * <> // end::mv_list[] +include::layout/mv_append.asciidoc[] include::layout/mv_avg.asciidoc[] include::layout/mv_concat.asciidoc[] include::layout/mv_count.asciidoc[] diff --git a/docs/reference/how-to/indexing-speed.asciidoc b/docs/reference/how-to/indexing-speed.asciidoc index 2bff5f82bf736..12de469c68449 100644 --- a/docs/reference/how-to/indexing-speed.asciidoc +++ b/docs/reference/how-to/indexing-speed.asciidoc @@ -94,6 +94,7 @@ auto-generated ids, Elasticsearch can skip this check, which makes indexing faster. [discrete] +[[indexing-use-faster-hardware]] === Use faster hardware If indexing is I/O-bound, consider increasing the size of the filesystem cache @@ -110,13 +111,10 @@ different nodes so there's redundancy for any node failures. You can also use <> to backup the index for further insurance. -Directly-attached (local) storage generally performs better than remote storage -because it is simpler to configure well and avoids communications overheads. -With careful tuning it is sometimes possible to achieve acceptable performance -using remote storage too. Benchmark your system with a realistic workload to -determine the effects of any tuning parameters. If you cannot achieve the -performance you expect, work with the vendor of your storage system to identify -the problem. +[discrete] +==== Local vs.remote storage + +include::./remote-storage.asciidoc[] [discrete] === Indexing buffer size diff --git a/docs/reference/how-to/remote-storage.asciidoc b/docs/reference/how-to/remote-storage.asciidoc new file mode 100644 index 0000000000000..e652d7eb5fdbf --- /dev/null +++ b/docs/reference/how-to/remote-storage.asciidoc @@ -0,0 +1,11 @@ +Directly-attached (local) storage generally performs +better than remote storage because it is simpler to configure well and avoids +communications overheads. + +Some remote storage performs very poorly, especially +under the kind of load that {es} imposes. However, with careful tuning, it is +sometimes possible to achieve acceptable performance using remote storage too. +Before committing to a particular storage architecture, benchmark your system +with a realistic workload to determine the effects of any tuning parameters. If +you cannot achieve the performance you expect, work with the vendor of your +storage system to identify the problem. \ No newline at end of file diff --git a/docs/reference/how-to/search-speed.asciidoc b/docs/reference/how-to/search-speed.asciidoc index 0db3ca04e99a7..0ef55d7808873 100644 --- a/docs/reference/how-to/search-speed.asciidoc +++ b/docs/reference/how-to/search-speed.asciidoc @@ -38,6 +38,7 @@ for `/dev/nvme0n1`, specify `blockdev --setra 256 /dev/nvme0n1`. // end::readahead[] [discrete] +[[search-use-faster-hardware]] === Use faster hardware If your searches are I/O-bound, consider increasing the size of the filesystem @@ -46,16 +47,13 @@ sequential and random reads across multiple files, and there may be many searches running concurrently on each shard, so SSD drives tend to perform better than spinning disks. -Directly-attached (local) storage generally performs better than remote storage -because it is simpler to configure well and avoids communications overheads. -With careful tuning it is sometimes possible to achieve acceptable performance -using remote storage too. Benchmark your system with a realistic workload to -determine the effects of any tuning parameters. If you cannot achieve the -performance you expect, work with the vendor of your storage system to identify -the problem. - If your searches are CPU-bound, consider using a larger number of faster CPUs. +[discrete] +==== Local vs. remote storage + +include::./remote-storage.asciidoc[] + [discrete] === Document modeling diff --git a/docs/reference/how-to/shard-limits.asciidoc b/docs/reference/how-to/shard-limits.asciidoc new file mode 100644 index 0000000000000..1127c8e7213de --- /dev/null +++ b/docs/reference/how-to/shard-limits.asciidoc @@ -0,0 +1,4 @@ +<> prevent creation of more than +1000 non-frozen shards per node, and 3000 frozen shards per dedicated frozen +node. Make sure you have enough nodes of each type in your cluster to handle +the number of shards you need. \ No newline at end of file diff --git a/docs/reference/how-to/size-your-shards.asciidoc b/docs/reference/how-to/size-your-shards.asciidoc index 4e2e9e0061b31..56e5fbbf15c77 100644 --- a/docs/reference/how-to/size-your-shards.asciidoc +++ b/docs/reference/how-to/size-your-shards.asciidoc @@ -34,6 +34,9 @@ cluster sizing video]. As you test different shard configurations, use {kib}'s {kibana-ref}/elasticsearch-metrics.html[{es} monitoring tools] to track your cluster's stability and performance. +The performance of an {es} node is often limited by the performance of the underlying storage. +Review our recommendations for optimizing your storage for <> and <>. + The following sections provide some reminders and guidelines you should consider when designing your sharding strategy. If your cluster is already oversharded, see <>. @@ -225,10 +228,7 @@ GET _cat/shards?v=true [[shard-count-per-node-recommendation]] ==== Add enough nodes to stay within the cluster shard limits -The <> prevent creation of more than -1000 non-frozen shards per node, and 3000 frozen shards per dedicated frozen -node. Make sure you have enough nodes of each type in your cluster to handle -the number of shards you need. +include::./shard-limits.asciidoc[] [discrete] [[field-count-recommendation]] diff --git a/docs/reference/indices/get-data-stream.asciidoc b/docs/reference/indices/get-data-stream.asciidoc index 0a318cd135914..b88a1a1be2a7e 100644 --- a/docs/reference/indices/get-data-stream.asciidoc +++ b/docs/reference/indices/get-data-stream.asciidoc @@ -358,4 +358,4 @@ The API returns the following response: // TESTRESPONSE[s/"index_name": ".ds-my-data-stream-two-2099.03.08-000001"/"index_name": $body.data_streams.1.indices.0.index_name/] // TESTRESPONSE[s/"index_uuid": "3liBu2SYS5axasRt6fUIpA"/"index_uuid": $body.data_streams.1.indices.0.index_uuid/] // TESTRESPONSE[s/"status": "GREEN"/"status": "YELLOW"/] -// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": false}/] +// TESTRESPONSE[s/"replicated": false/"replicated": false,"failure_store":{"enabled": false, "indices": [], "rollover_on_write": true}/] diff --git a/docs/reference/mapping/types/dense-vector.asciidoc b/docs/reference/mapping/types/dense-vector.asciidoc index 6294423985ec6..8759059a319da 100644 --- a/docs/reference/mapping/types/dense-vector.asciidoc +++ b/docs/reference/mapping/types/dense-vector.asciidoc @@ -115,13 +115,23 @@ that sacrifices result accuracy for improved speed. ==== Automatically quantize vectors for kNN search The `dense_vector` type supports quantization to reduce the memory footprint required when <> `float` vectors. -Currently the only quantization method supported is `int8` and provided vectors `element_type` must be `float`. To use -a quantized index, you can set your index type to `int8_hnsw`. When indexing `float` vectors, the current default +The two following quantization strategies are supported: + ++ +-- +`int8` - Quantizes each dimension of the vector to 1-byte integers. This can reduce the memory footprint by 75% at the cost of some accuracy. +`int4` - Quantizes each dimension of the vector to half-byte integers. This can reduce the memory footprint by 87% at the cost of some accuracy. +-- + +To use a quantized index, you can set your index type to `int8_hnsw` or `int4_hnsw`. When indexing `float` vectors, the current default index type is `int8_hnsw`. -When using the `int8_hnsw` index, each of the `float` vectors' dimensions are quantized to 1-byte integers. This can -reduce the memory footprint by as much as 75% at the cost of some accuracy. However, the disk usage can increase by -25% due to the overhead of storing the quantized and raw vectors. +NOTE: Quantization will continue to keep the raw float vector values on disk for reranking, reindexing, and quantization improvements over the lifetime of the data. +This means disk usage will increase by ~25% for `int8` and ~12.5% for `int4` due to the overhead of storing the quantized and raw vectors. + +NOTE: `int4` quantization requires an even number of vector dimensions. + +Here is an example of how to create a byte-quantized index: [source,console] -------------------------------------------------- @@ -142,6 +152,27 @@ PUT my-byte-quantized-index } -------------------------------------------------- +Here is an example of how to create a half-byte-quantized index: + +[source,console] +-------------------------------------------------- +PUT my-byte-quantized-index +{ + "mappings": { + "properties": { + "my_vector": { + "type": "dense_vector", + "dims": 4, + "index": true, + "index_options": { + "type": "int4_hnsw" + } + } + } + } +} +-------------------------------------------------- + [role="child_attributes"] [[dense-vector-params]] ==== Parameters for dense vector fields @@ -247,27 +278,34 @@ The type of kNN algorithm to use. Can be either any of: This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically scalar quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint by 4x at the cost of some accuracy. See <>. +* `int4_hnsw` - This utilizes the https://arxiv.org/abs/1603.09320[HNSW algorithm] in addition to automatically scalar +quantization for scalable approximate kNN search with `element_type` of `float`. This can reduce the memory footprint +by 8x at the cost of some accuracy. See <>. * `flat` - This utilizes a brute-force search algorithm for exact kNN search. This supports all `element_type` values. * `int8_flat` - This utilizes a brute-force search algorithm in addition to automatically scalar quantization. Only supports `element_type` of `float`. +* `int4_flat` - This utilizes a brute-force search algorithm in addition to automatically half-byte scalar quantization. Only supports +`element_type` of `float`. -- `m`::: (Optional, integer) The number of neighbors each node will be connected to in the HNSW graph. -Defaults to `16`. Only applicable to `hnsw` and `int8_hnsw` index types. +Defaults to `16`. Only applicable to `hnsw`, `int8_hnsw`, and `int4_hnsw` index types. `ef_construction`::: (Optional, integer) The number of candidates to track while assembling the list of nearest -neighbors for each new node. Defaults to `100`. Only applicable to `hnsw` and `int8_hnsw` index types. +neighbors for each new node. Defaults to `100`. Only applicable to `hnsw`, `int8_hnsw`, and `int4_hnsw` index types. `confidence_interval`::: (Optional, float) -Only applicable to `int8_hnsw` and `int8_flat` index types. The confidence interval to use when quantizing the vectors, -can be any value between and including `0.90` and `1.0`. This value restricts the values used when calculating -the quantization thresholds. For example, a value of `0.95` will only use the middle 95% of the values when -calculating the quantization thresholds (e.g. the highest and lowest 2.5% of values will be ignored). -Defaults to `1/(dims + 1)`. +Only applicable to `int8_hnsw`, `int4_hnsw`, `int8_flat`, and `int4_flat` index types. The confidence interval to use when quantizing the vectors. +Can be any value between and including `0.90` and `1.0` or exactly `0`. When the value is `0`, this indicates that dynamic +quantiles should be calculated for optimized quantization. When between `0.90` and `1.0`, +this value restricts the values used when calculating the quantization thresholds. +For example, a value of `0.95` will only use the middle 95% of the values when calculating the quantization thresholds +(e.g. the highest and lowest 2.5% of values will be ignored). +Defaults to `1/(dims + 1)` for `int8` quantized vectors and `0` for `int4` for dynamic quantile calculation. ==== [[dense-vector-synthetic-source]] diff --git a/docs/reference/mapping/types/geo-point.asciidoc b/docs/reference/mapping/types/geo-point.asciidoc index aab40efd15acf..6db05188dfb98 100644 --- a/docs/reference/mapping/types/geo-point.asciidoc +++ b/docs/reference/mapping/types/geo-point.asciidoc @@ -220,8 +220,7 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. `geo_point` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -<>, <>, or with +default configuration. Synthetic `_source` cannot be used together with <> or with <> disabled. Synthetic source always sorts `geo_point` fields (first by latitude and then diff --git a/docs/reference/mapping/types/histogram.asciidoc b/docs/reference/mapping/types/histogram.asciidoc index 3e221b11182ad..8cd30110250bf 100644 --- a/docs/reference/mapping/types/histogram.asciidoc +++ b/docs/reference/mapping/types/histogram.asciidoc @@ -79,8 +79,7 @@ any issues, but features in technical preview are not subject to the support SLA of official GA features. `histogram` fields support <> in their -default configuration. Synthetic `_source` cannot be used together with -<> or <>. +default configuration. Synthetic `_source` cannot be used together with <>. NOTE: To save space, zero-count buckets are not stored in the histogram doc values. As a result, when indexing a histogram field in an index with synthetic source enabled, diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index 1e87faea5b13a..d1e1c037e571e 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -249,9 +249,9 @@ be changed or removed in a future release. Elastic will work to fix any issues, but features in technical preview are not subject to the support SLA of official GA features. -All numeric fields except `unsigned_long` support <> in their default configuration. Synthetic `_source` cannot be used -together with <>, <>, or +together with <>, or with <> disabled. Synthetic source always sorts numeric fields. For example: diff --git a/docs/reference/mapping/types/semantic-text.asciidoc b/docs/reference/mapping/types/semantic-text.asciidoc index 454eefd20b07f..bbb501c4ccc36 100644 --- a/docs/reference/mapping/types/semantic-text.asciidoc +++ b/docs/reference/mapping/types/semantic-text.asciidoc @@ -139,3 +139,12 @@ the resulting embeddings. This imposes a restriction on bulk updates to documents with `semantic_text`. In bulk requests, all fields that are copied to a `semantic_text` field must have a value to ensure every embedding is calculated correctly. + +[discrete] +[[limitations]] +==== Limitations + +`semantic_text` field types have the following limitations: + +* `semantic_text` fields are not currently supported as elements of <>. +* `semantic_text` fields can't be defined as <> of another field, nor can they contain other fields as multi-fields. diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 6bbc98db1c2e1..a69fd2f1812e9 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -430,16 +430,16 @@ end::daily-model-snapshot-retention-after-days[] tag::data-description[] The data description defines the format of the input data when you send data to -the job by using the <> API. Note that when using a -{dfeed}, only the `time_field` needs to be set, the rest of the properties are -automatically set. When data is received via the <> API, +the job by using the <> API. Note that when using a +{dfeed}, only the `time_field` needs to be set, the rest of the properties are +automatically set. When data is received via the <> API, it is not stored in {es}. Only the results for {anomaly-detect} are retained. + .Properties of `data_description` [%collapsible%open] ==== `format`::: - (string) Only `xcontent` format is supported at this time, and this is the + (string) Only `xcontent` format is supported at this time, and this is the default value. `time_field`::: @@ -1285,6 +1285,10 @@ tag::job-id-datafeed[] The unique identifier for the job to which the {dfeed} sends data. end::job-id-datafeed[] +tag::output-memory-allocator-bytes[] +The amount of memory, in bytes, used to output {anomaly-job} documents. +end::output-memory-allocator-bytes[] + tag::lambda[] Advanced configuration option. Regularization parameter to prevent overfitting on the training data set. Multiplies an L2 regularization term which applies to diff --git a/docs/reference/modules/discovery/quorums.asciidoc b/docs/reference/modules/discovery/quorums.asciidoc index beee6da60231e..f6f50b88b3190 100644 --- a/docs/reference/modules/discovery/quorums.asciidoc +++ b/docs/reference/modules/discovery/quorums.asciidoc @@ -9,7 +9,7 @@ succeeded on receipt of responses from a _quorum_, which is a subset of the master-eligible nodes in the cluster. The advantage of requiring only a subset of the nodes to respond is that it means some of the nodes can fail without preventing the cluster from making progress. The quorums are carefully chosen so -the cluster does not have a "{wikipedia}/Split-brain_(computing)[split-brain]" scenario where it's partitioned into +the cluster does not have a "split brain" scenario where it's partitioned into two pieces such that each piece may make decisions that are inconsistent with those of the other piece. diff --git a/docs/reference/modules/node.asciidoc b/docs/reference/modules/node.asciidoc index 81df2cf4a2a6c..022e8b5d1e2fe 100644 --- a/docs/reference/modules/node.asciidoc +++ b/docs/reference/modules/node.asciidoc @@ -1,5 +1,5 @@ [[modules-node]] -=== Node +=== Nodes Any time that you start an instance of {es}, you are starting a _node_. A collection of connected nodes is called a <>. If you @@ -14,6 +14,10 @@ All nodes know about all the other nodes in the cluster and can forward client requests to the appropriate node. // end::modules-node-description-tag[] +TIP: The performance of an {es} node is often limited by the performance of the underlying storage. +Review our recommendations for optimizing your storage for <> and +<>. + [[node-roles]] ==== Node roles @@ -236,6 +240,8 @@ assign data nodes to specific tiers: `data_content`,`data_hot`, `data_warm`, If you want to include a node in all tiers, or if your cluster does not use multiple tiers, then you can use the generic `data` role. +include::../how-to/shard-limits.asciidoc[] + WARNING: If you assign a node to a specific tier using a specialized data role, then you shouldn't also assign it the generic `data` role. The generic `data` role takes precedence over specialized data roles. [[generic-data-node]] @@ -471,12 +477,6 @@ properly-configured remote block devices (e.g. a SAN) and remote filesystems storage. You can run multiple {es} nodes on the same filesystem, but each {es} node must have its own data path. -The performance of an {es} cluster is often limited by the performance of the -underlying storage, so you must ensure that your storage supports acceptable -performance. Some remote storage performs very poorly, especially under the -kind of load that {es} imposes, so make sure to benchmark your system carefully -before committing to a particular storage architecture. - TIP: When using the `.zip` or `.tar.gz` distributions, the `path.data` setting should be configured to locate the data directory outside the {es} home directory, so that the home directory can be deleted without deleting your data! diff --git a/docs/reference/release-notes.asciidoc b/docs/reference/release-notes.asciidoc index 9b1257e9054c9..2e043834c9969 100644 --- a/docs/reference/release-notes.asciidoc +++ b/docs/reference/release-notes.asciidoc @@ -7,6 +7,7 @@ This section summarizes the changes in each release. * <> +* <> * <> * <> * <> @@ -68,6 +69,7 @@ This section summarizes the changes in each release. -- include::release-notes/8.15.0.asciidoc[] +include::release-notes/8.14.1.asciidoc[] include::release-notes/8.14.0.asciidoc[] include::release-notes/8.13.4.asciidoc[] include::release-notes/8.13.3.asciidoc[] diff --git a/docs/reference/release-notes/8.14.1.asciidoc b/docs/reference/release-notes/8.14.1.asciidoc new file mode 100644 index 0000000000000..f161c7d08099c --- /dev/null +++ b/docs/reference/release-notes/8.14.1.asciidoc @@ -0,0 +1,36 @@ +[[release-notes-8.14.1]] +== {es} version 8.14.1 + + +Also see <>. + +[[bug-8.14.1]] +[float] +=== Bug fixes + +Authorization:: +* Fix task cancellation authz on fulfilling cluster {es-pull}109357[#109357] + +Infra/Core:: +* Guard systemd library lookup from unreadable directories {es-pull}108931[#108931] + +Machine Learning:: +* Reset retryable index requests after failures {es-pull}109320[#109320] + +Network:: +* Fix task cancellation on remote cluster when original request fails {es-pull}109440[#109440] + +Transform:: +* Reset max page size to settings value {es-pull}109532[#109532] (issue: {es-issue}109308[#109308]) + +Vector Search:: +* Correct how hex strings are handled when dynamically updating vector dims {es-pull}109423[#109423] + +[[enhancement-8.14.1]] +[float] +=== Enhancements + +Infra/Settings:: +* Add remove index setting command {es-pull}109276[#109276] + + diff --git a/docs/reference/release-notes/highlights.asciidoc b/docs/reference/release-notes/highlights.asciidoc index e6016fe438e24..ead1596c64fdd 100644 --- a/docs/reference/release-notes/highlights.asciidoc +++ b/docs/reference/release-notes/highlights.asciidoc @@ -73,3 +73,16 @@ databases from MaxMind. {es-pull}108683[#108683] +[discrete] +[[update_elasticsearch_to_lucene_9_11]] +=== Update Elasticsearch to Lucene 9.11 +Elasticsearch is now updated using the latest Lucene version 9.11. +Here are the full release notes: +But, here are some particular highlights: +- Usage of MADVISE for better memory management: https://github.com/apache/lucene/pull/13196 +- Use RWLock to access LRUQueryCache to reduce contention: https://github.com/apache/lucene/pull/13306 +- Speedup multi-segment HNSW graph search for nested kNN queries: https://github.com/apache/lucene/pull/13121 +- Add a MemorySegment Vector scorer - for scoring without copying on-heap vectors: https://github.com/apache/lucene/pull/13339 + +{es-pull}109219[#109219] + diff --git a/docs/reference/rest-api/common-parms.asciidoc b/docs/reference/rest-api/common-parms.asciidoc index 15414dde86e52..e537fc959965a 100644 --- a/docs/reference/rest-api/common-parms.asciidoc +++ b/docs/reference/rest-api/common-parms.asciidoc @@ -507,6 +507,10 @@ Return all statistics. `completion`:: <> statistics. +`dense_vector`:: +Total number of dense vectors indexed. +<> can affect this statistic. + `docs`:: Number of documents, number of deleted docs which have not yet merged out, and total size in bytes. <> can affect this statistic. @@ -551,11 +555,17 @@ If the `include_segment_file_sizes` parameter is `true`, this metric includes the aggregated disk usage of each Lucene index file. +`sparse_vector`:: +Total number of sparse vectors indexed. + +<> can affect this statistic. + `store`:: Size of the index in <>. `translog`:: <> statistics. + -- end::index-metric[] @@ -606,7 +616,6 @@ the request. You must provide either a `query_vector_builder` or `query_vector`, but not both. Refer to <> to learn more. end::knn-query-vector-builder[] - tag::knn-similarity[] The minimum similarity required for a document to be considered a match. The similarity value calculated relates to the raw <> used. Not the @@ -1224,11 +1233,9 @@ Can also be set to `-1` to indicate that the request should never timeout. end::master-timeout[] `timeout`:: -(Optional, <>) -Period to wait for a response from all relevant nodes in the cluster after -updating the cluster metadata. If no response is received before the timeout -expires, the cluster metadata update still applies but the response will -indicate that it was not completely acknowledged. Defaults to `30s`. +(Optional, <>) Period to wait for a response from all relevant nodes in the cluster after updating the cluster metadata. +If no response is received before the timeout expires, the cluster metadata update still applies but the response will indicate that it was not completely acknowledged. +Defaults to `30s`. Can also be set to `-1` to indicate that the request should never timeout. end::timeoutparms[] diff --git a/docs/reference/rest-api/security/query-api-key.asciidoc b/docs/reference/rest-api/security/query-api-key.asciidoc index 5760968cc7e9a..513cb99a55a4c 100644 --- a/docs/reference/rest-api/security/query-api-key.asciidoc +++ b/docs/reference/rest-api/security/query-api-key.asciidoc @@ -177,7 +177,7 @@ The query supports a subset of query types, including <>, <>, <>, <>, <>, <>, -and <> +and <>. + You can query the following public values associated with an API key. + diff --git a/docs/reference/rest-api/security/query-user.asciidoc b/docs/reference/rest-api/security/query-user.asciidoc index d0c7b44faf3bd..952e0f40f2a3a 100644 --- a/docs/reference/rest-api/security/query-user.asciidoc +++ b/docs/reference/rest-api/security/query-user.asciidoc @@ -40,7 +40,10 @@ You can specify the following parameters in the request body: The query supports a subset of query types, including <>, <>, <>, <>, -<>, <> and <>. +<>, <>, +<>, <>, +<>, <>, +and <>. + You can query the following public values associated with a user. + diff --git a/docs/reference/search/search-your-data/knn-search.asciidoc b/docs/reference/search/search-your-data/knn-search.asciidoc index db4b0febb07ba..0e61b44eda413 100644 --- a/docs/reference/search/search-your-data/knn-search.asciidoc +++ b/docs/reference/search/search-your-data/knn-search.asciidoc @@ -274,7 +274,7 @@ in the index. NOTE: The default index type for `dense_vector` is `int8_hnsw`. -To use quantization, you can use the index type `int8_hnsw` object in the `dense_vector` mapping. +To use quantization, you can use the index type `int8_hnsw` or `int4_hnsw` object in the `dense_vector` mapping. [source,console] ---- diff --git a/docs/reference/settings/data-stream-lifecycle-settings.asciidoc b/docs/reference/settings/data-stream-lifecycle-settings.asciidoc index 023a8fcf860eb..0f00e956472d0 100644 --- a/docs/reference/settings/data-stream-lifecycle-settings.asciidoc +++ b/docs/reference/settings/data-stream-lifecycle-settings.asciidoc @@ -6,8 +6,6 @@ Data stream lifecycle settings ++++ -preview::[] - These are the settings available for configuring <>. ==== Cluster level settings diff --git a/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc b/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc index d1431b8cb6706..8824977d660e4 100644 --- a/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc +++ b/docs/reference/snapshot-restore/apis/delete-snapshot-api.asciidoc @@ -58,6 +58,11 @@ Comma-separated list of snapshot names to delete. Also accepts wildcards (`*`). include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout] +`wait_for_completion`:: +(Optional, Boolean) If `true`, the request returns a response when the matching +snapshots are all deleted. If `false`, the request returns a response as soon as +the deletes are scheduled. Defaults to `true`. + [[delete-snapshot-api-example]] ==== {api-example-title} diff --git a/docs/reference/vectors/vector-functions.asciidoc b/docs/reference/vectors/vector-functions.asciidoc index e0ed85189c97d..4e627ef18ec6c 100644 --- a/docs/reference/vectors/vector-functions.asciidoc +++ b/docs/reference/vectors/vector-functions.asciidoc @@ -12,9 +12,10 @@ This is the list of available vector functions and vector access methods: 1. <> – calculates cosine similarity 2. <> – calculates dot product 3. <> – calculates L^1^ distance -4. <> - calculates L^2^ distance -5. <].vectorValue`>> – returns a vector's value as an array of floats -6. <].magnitude`>> – returns a vector's magnitude +4. <> – calculates Hamming distance +5. <> - calculates L^2^ distance +6. <].vectorValue`>> – returns a vector's value as an array of floats +7. <].magnitude`>> – returns a vector's magnitude NOTE: The recommended way to access dense vectors is through the `cosineSimilarity`, `dotProduct`, `l1norm` or `l2norm` functions. Please note @@ -35,8 +36,15 @@ PUT my-index-000001 "properties": { "my_dense_vector": { "type": "dense_vector", + "index": false, "dims": 3 }, + "my_byte_dense_vector": { + "type": "dense_vector", + "index": false, + "dims": 3, + "element_type": "byte" + }, "status" : { "type" : "keyword" } @@ -47,12 +55,14 @@ PUT my-index-000001 PUT my-index-000001/_doc/1 { "my_dense_vector": [0.5, 10, 6], + "my_byte_dense_vector": [0, 10, 6], "status" : "published" } PUT my-index-000001/_doc/2 { "my_dense_vector": [-0.5, 10, 10], + "my_byte_dense_vector": [0, 10, 10], "status" : "published" } @@ -179,6 +189,40 @@ we reversed the output from `l1norm` and `l2norm`. Also, to avoid division by 0 when a document vector matches the query exactly, we added `1` in the denominator. +[[vector-functions-hamming]] +====== Hamming distance + +The `hamming` function calculates {wikipedia}/Hamming_distance[Hamming distance] between a given query vector and +document vectors. It is only available for byte vectors. + +[source,console] +-------------------------------------------------- +GET my-index-000001/_search +{ + "query": { + "script_score": { + "query" : { + "bool" : { + "filter" : { + "term" : { + "status" : "published" + } + } + } + }, + "script": { + "source": "(24 - hamming(params.queryVector, 'my_byte_dense_vector')) / 24", <1> + "params": { + "queryVector": [4, 3, 0] + } + } + } + } +} +-------------------------------------------------- + +<1> Calculate the Hamming distance and normalize it by the bits to get a score between 0 and 1. + [[vector-functions-l2]] ====== L^2^ distance (Euclidean distance) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 9860253d70e58..6e4beb0953b56 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -266,6 +266,11 @@ + + + + + @@ -286,6 +291,11 @@ + + + + + @@ -331,6 +341,11 @@ + + + + + @@ -346,6 +361,11 @@ + + + + + @@ -366,6 +386,11 @@ + + + + + diff --git a/libs/logstash-bridge/README.md b/libs/logstash-bridge/README.md new file mode 100644 index 0000000000000..dd629724878b5 --- /dev/null +++ b/libs/logstash-bridge/README.md @@ -0,0 +1,8 @@ +## Logstash Bridge + +This package contains bridge functionality to ensure that Logstash's Elastic Integration plugin +has access to the minimal subset of Elasticsearch to perform its functions without relying on +other Elasticsearch internals. + +If a change is introduced in a separate Elasticsearch project that causes this project to fail, +please consult with members of @elastic/logstash to chart a path forward. diff --git a/libs/logstash-bridge/build.gradle b/libs/logstash-bridge/build.gradle new file mode 100644 index 0000000000000..28fd6149fd7d8 --- /dev/null +++ b/libs/logstash-bridge/build.gradle @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.build' + +dependencies { + compileOnly project(':server') + compileOnly project(':libs:elasticsearch-core') + compileOnly project(':libs:elasticsearch-plugin-api') + compileOnly project(':libs:elasticsearch-x-content') + compileOnly project(':modules:lang-painless') + compileOnly project(':modules:lang-painless:spi') + compileOnly project(':modules:lang-mustache') + compileOnly project(':modules:ingest-common') +// compileOnly project(':modules:ingest-geoip') +} + +tasks.named('forbiddenApisMain').configure { + replaceSignatureFiles 'jdk-signatures' +} diff --git a/libs/logstash-bridge/src/main/java/module-info.java b/libs/logstash-bridge/src/main/java/module-info.java new file mode 100644 index 0000000000000..49b0e13c14cd4 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/module-info.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/** Elasticsearch Logstash Bridge. */ +module org.elasticsearch.logstashbridge { + requires org.elasticsearch.base; + requires org.elasticsearch.grok; + requires org.elasticsearch.server; + requires org.elasticsearch.painless; + requires org.elasticsearch.painless.spi; + requires org.elasticsearch.mustache; + requires org.elasticsearch.xcontent; + + exports org.elasticsearch.logstashbridge; + exports org.elasticsearch.logstashbridge.common; + exports org.elasticsearch.logstashbridge.core; + exports org.elasticsearch.logstashbridge.env; + exports org.elasticsearch.logstashbridge.ingest; + exports org.elasticsearch.logstashbridge.plugins; + exports org.elasticsearch.logstashbridge.script; + exports org.elasticsearch.logstashbridge.threadpool; +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/StableBridgeAPI.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/StableBridgeAPI.java new file mode 100644 index 0000000000000..cdf2ab4ee7be3 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/StableBridgeAPI.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge; + +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * A {@code StableBridgeAPI} is the stable bridge to an Elasticsearch API, and can produce instances + * from the actual API that they mirror. As part of the LogstashBridge project, these classes are relied + * upon by the "Elastic Integration Filter Plugin" for Logstash and their external shapes mut not change + * without coordination with the maintainers of that project. + * + * @param the actual type of the Elasticsearch API being mirrored + */ +public interface StableBridgeAPI { + T unwrap(); + + static T unwrapNullable(final StableBridgeAPI nullableStableBridgeAPI) { + if (Objects.isNull(nullableStableBridgeAPI)) { + return null; + } + return nullableStableBridgeAPI.unwrap(); + } + + static Map unwrap(final Map> bridgeMap) { + return bridgeMap.entrySet().stream().collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> e.getValue().unwrap())); + } + + static > Map wrap(final Map rawMap, final Function wrapFunction) { + return rawMap.entrySet().stream().collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e -> wrapFunction.apply(e.getValue()))); + } + + static > B wrap(final T delegate, final Function wrapFunction) { + if (Objects.isNull(delegate)) { + return null; + } + return wrapFunction.apply(delegate); + } + + abstract class Proxy implements StableBridgeAPI { + protected final T delegate; + + protected Proxy(final T delegate) { + this.delegate = delegate; + } + + @Override + public T unwrap() { + return delegate; + } + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/common/SettingsBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/common/SettingsBridge.java new file mode 100644 index 0000000000000..86fd0fcf75658 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/common/SettingsBridge.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.common; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.logstashbridge.StableBridgeAPI; + +public class SettingsBridge extends StableBridgeAPI.Proxy { + + public static SettingsBridge wrap(final Settings delegate) { + return new SettingsBridge(delegate); + } + + public static Builder builder() { + return Builder.wrap(Settings.builder()); + } + + public SettingsBridge(final Settings delegate) { + super(delegate); + } + + @Override + public Settings unwrap() { + return this.delegate; + } + + public static class Builder extends StableBridgeAPI.Proxy { + static Builder wrap(final Settings.Builder delegate) { + return new Builder(delegate); + } + + private Builder(final Settings.Builder delegate) { + super(delegate); + } + + public Builder put(final String key, final String value) { + this.delegate.put(key, value); + return this; + } + + public SettingsBridge build() { + return new SettingsBridge(this.delegate.build()); + } + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/core/IOUtilsBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/core/IOUtilsBridge.java new file mode 100644 index 0000000000000..810c671e5b8eb --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/core/IOUtilsBridge.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.core; + +import org.elasticsearch.core.IOUtils; + +import java.io.Closeable; + +public class IOUtilsBridge { + public static void closeWhileHandlingException(final Iterable objects) { + IOUtils.closeWhileHandlingException(objects); + } + + public static void closeWhileHandlingException(final Closeable closeable) { + IOUtils.closeWhileHandlingException(closeable); + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/env/EnvironmentBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/env/EnvironmentBridge.java new file mode 100644 index 0000000000000..8ae3ce2d33d28 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/env/EnvironmentBridge.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.env; + +import org.elasticsearch.env.Environment; +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.logstashbridge.common.SettingsBridge; + +import java.nio.file.Path; + +public class EnvironmentBridge extends StableBridgeAPI.Proxy { + public static EnvironmentBridge wrap(final Environment delegate) { + return new EnvironmentBridge(delegate); + } + + public EnvironmentBridge(final SettingsBridge settingsBridge, final Path configPath) { + this(new Environment(settingsBridge.unwrap(), configPath)); + } + + private EnvironmentBridge(final Environment delegate) { + super(delegate); + } + + @Override + public Environment unwrap() { + return this.delegate; + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ConfigurationUtilsBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ConfigurationUtilsBridge.java new file mode 100644 index 0000000000000..2d7f5c27b16e0 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ConfigurationUtilsBridge.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.ingest; + +import org.elasticsearch.ingest.ConfigurationUtils; +import org.elasticsearch.logstashbridge.script.ScriptServiceBridge; +import org.elasticsearch.logstashbridge.script.TemplateScriptBridge; + +import java.util.Map; + +public class ConfigurationUtilsBridge { + public static TemplateScriptBridge.Factory compileTemplate( + final String processorType, + final String processorTag, + final String propertyName, + final String propertyValue, + final ScriptServiceBridge scriptServiceBridge + ) { + return new TemplateScriptBridge.Factory( + ConfigurationUtils.compileTemplate(processorType, processorTag, propertyName, propertyValue, scriptServiceBridge.unwrap()) + ); + } + + public static String readStringProperty( + final String processorType, + final String processorTag, + final Map configuration, + final String propertyName + ) { + return ConfigurationUtils.readStringProperty(processorType, processorTag, configuration, propertyName); + } + + public static Boolean readBooleanProperty( + final String processorType, + final String processorTag, + final Map configuration, + final String propertyName, + final boolean defaultValue + ) { + return ConfigurationUtils.readBooleanProperty(processorType, processorTag, configuration, propertyName, defaultValue); + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java new file mode 100644 index 0000000000000..5135034485392 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/IngestDocumentBridge.java @@ -0,0 +1,91 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.ingest; + +import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.LogstashInternalBridge; +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.logstashbridge.script.MetadataBridge; +import org.elasticsearch.logstashbridge.script.TemplateScriptBridge; + +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +public class IngestDocumentBridge extends StableBridgeAPI.Proxy { + + public static String INGEST_KEY = IngestDocument.INGEST_KEY; + + public static IngestDocumentBridge wrap(final IngestDocument ingestDocument) { + if (ingestDocument == null) { + return null; + } + return new IngestDocumentBridge(ingestDocument); + } + + public IngestDocumentBridge(final Map sourceAndMetadata, final Map ingestMetadata) { + this(new IngestDocument(sourceAndMetadata, ingestMetadata)); + } + + private IngestDocumentBridge(IngestDocument inner) { + super(inner); + } + + public MetadataBridge getMetadata() { + return new MetadataBridge(delegate.getMetadata()); + } + + public Map getSource() { + return delegate.getSource(); + } + + public boolean updateIndexHistory(final String index) { + return delegate.updateIndexHistory(index); + } + + public Set getIndexHistory() { + return Set.copyOf(delegate.getIndexHistory()); + } + + public boolean isReroute() { + return LogstashInternalBridge.isReroute(delegate); + } + + public void resetReroute() { + LogstashInternalBridge.resetReroute(delegate); + } + + public Map getIngestMetadata() { + return Map.copyOf(delegate.getIngestMetadata()); + } + + public T getFieldValue(final String fieldName, final Class type) { + return delegate.getFieldValue(fieldName, type); + } + + public T getFieldValue(final String fieldName, final Class type, final boolean ignoreMissing) { + return delegate.getFieldValue(fieldName, type, ignoreMissing); + } + + public String renderTemplate(final TemplateScriptBridge.Factory templateScriptFactory) { + return delegate.renderTemplate(templateScriptFactory.unwrap()); + } + + public void setFieldValue(final String path, final Object value) { + delegate.setFieldValue(path, value); + } + + public void removeField(final String path) { + delegate.removeField(path); + } + + // public void executePipeline(Pipeline pipeline, BiConsumer handler) { + public void executePipeline(final PipelineBridge pipelineBridge, final BiConsumer handler) { + this.delegate.executePipeline(pipelineBridge.unwrap(), (unwrapped, e) -> handler.accept(IngestDocumentBridge.wrap(unwrapped), e)); + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java new file mode 100644 index 0000000000000..835e377c71b31 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineBridge.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.ingest; + +import org.elasticsearch.ingest.Pipeline; +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.logstashbridge.script.ScriptServiceBridge; + +import java.util.Map; +import java.util.function.BiConsumer; + +public class PipelineBridge extends StableBridgeAPI.Proxy { + public static PipelineBridge wrap(final Pipeline pipeline) { + return new PipelineBridge(pipeline); + } + + public static PipelineBridge create( + String id, + Map config, + Map processorFactories, + ScriptServiceBridge scriptServiceBridge + ) throws Exception { + return wrap( + Pipeline.create(id, config, StableBridgeAPI.unwrap(processorFactories), StableBridgeAPI.unwrapNullable(scriptServiceBridge)) + ); + } + + public PipelineBridge(final Pipeline delegate) { + super(delegate); + } + + public String getId() { + return delegate.getId(); + } + + public void execute(final IngestDocumentBridge ingestDocumentBridge, final BiConsumer handler) { + this.delegate.execute( + StableBridgeAPI.unwrapNullable(ingestDocumentBridge), + (unwrapped, e) -> handler.accept(IngestDocumentBridge.wrap(unwrapped), e) + ); + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineConfigurationBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineConfigurationBridge.java new file mode 100644 index 0000000000000..d2aff89d1f236 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineConfigurationBridge.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.ingest; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.ingest.PipelineConfiguration; +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.xcontent.XContentType; + +import java.util.Map; + +public class PipelineConfigurationBridge extends StableBridgeAPI.Proxy { + public PipelineConfigurationBridge(final PipelineConfiguration delegate) { + super(delegate); + } + + public PipelineConfigurationBridge(final String pipelineId, final String jsonEncodedConfig) { + this(new PipelineConfiguration(pipelineId, new BytesArray(jsonEncodedConfig), XContentType.JSON)); + } + + public String getId() { + return delegate.getId(); + } + + public Map getConfigAsMap() { + return delegate.getConfigAsMap(); + } + + @Override + public int hashCode() { + return delegate.hashCode(); + } + + @Override + public String toString() { + return delegate.toString(); + } + + @Override + public boolean equals(final Object obj) { + if (this == obj) { + return true; + } else if (obj instanceof PipelineConfigurationBridge other) { + return delegate.equals(other.delegate); + } else { + return false; + } + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java new file mode 100644 index 0000000000000..7b88b12eb3c1c --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/ProcessorBridge.java @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.ingest; + +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.ingest.IngestService; +import org.elasticsearch.ingest.Processor; +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.logstashbridge.env.EnvironmentBridge; +import org.elasticsearch.logstashbridge.script.ScriptServiceBridge; +import org.elasticsearch.logstashbridge.threadpool.ThreadPoolBridge; + +import java.util.Map; +import java.util.function.BiConsumer; + +public interface ProcessorBridge extends StableBridgeAPI { + String getType(); + + String getTag(); + + String getDescription(); + + boolean isAsync(); + + void execute(IngestDocumentBridge ingestDocumentBridge, BiConsumer handler) throws Exception; + + static ProcessorBridge wrap(final Processor delegate) { + return new Wrapped(delegate); + } + + class Wrapped extends StableBridgeAPI.Proxy implements ProcessorBridge { + public Wrapped(final Processor delegate) { + super(delegate); + } + + @Override + public String getType() { + return unwrap().getType(); + } + + @Override + public String getTag() { + return unwrap().getTag(); + } + + @Override + public String getDescription() { + return unwrap().getDescription(); + } + + @Override + public boolean isAsync() { + return unwrap().isAsync(); + } + + @Override + public void execute(final IngestDocumentBridge ingestDocumentBridge, final BiConsumer handler) + throws Exception { + delegate.execute( + StableBridgeAPI.unwrapNullable(ingestDocumentBridge), + (id, e) -> handler.accept(IngestDocumentBridge.wrap(id), e) + ); + } + } + + class Parameters extends StableBridgeAPI.Proxy { + + public Parameters( + final EnvironmentBridge environmentBridge, + final ScriptServiceBridge scriptServiceBridge, + final ThreadPoolBridge threadPoolBridge + ) { + this( + new Processor.Parameters( + environmentBridge.unwrap(), + scriptServiceBridge.unwrap(), + null, + threadPoolBridge.unwrap().getThreadContext(), + threadPoolBridge.unwrap()::relativeTimeInMillis, + (delay, command) -> threadPoolBridge.unwrap() + .schedule(command, TimeValue.timeValueMillis(delay), threadPoolBridge.unwrap().generic()), + null, + null, + threadPoolBridge.unwrap().generic()::execute, + IngestService.createGrokThreadWatchdog(environmentBridge.unwrap(), threadPoolBridge.unwrap()) + ) + ); + } + + private Parameters(final Processor.Parameters delegate) { + super(delegate); + } + + @Override + public Processor.Parameters unwrap() { + return this.delegate; + } + } + + interface Factory extends StableBridgeAPI { + ProcessorBridge create( + Map registry, + String processorTag, + String description, + Map config + ) throws Exception; + + static Factory wrap(final Processor.Factory delegate) { + return new Wrapped(delegate); + } + + @Override + default Processor.Factory unwrap() { + final Factory stableAPIFactory = this; + return (registry, tag, description, config) -> stableAPIFactory.create( + StableBridgeAPI.wrap(registry, Factory::wrap), + tag, + description, + config + ).unwrap(); + } + + class Wrapped extends StableBridgeAPI.Proxy implements Factory { + private Wrapped(final Processor.Factory delegate) { + super(delegate); + } + + @Override + public ProcessorBridge create( + final Map registry, + final String processorTag, + final String description, + final Map config + ) throws Exception { + return ProcessorBridge.wrap(this.delegate.create(StableBridgeAPI.unwrap(registry), processorTag, description, config)); + } + + @Override + public Processor.Factory unwrap() { + return this.delegate; + } + } + } + +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/plugins/IngestPluginBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/plugins/IngestPluginBridge.java new file mode 100644 index 0000000000000..a27eaa9063dda --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/plugins/IngestPluginBridge.java @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.plugins; + +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.logstashbridge.ingest.ProcessorBridge; +import org.elasticsearch.plugins.IngestPlugin; + +import java.io.Closeable; +import java.io.IOException; +import java.util.Map; + +public interface IngestPluginBridge { + Map getProcessors(ProcessorBridge.Parameters parameters); + + static Wrapped wrap(final IngestPlugin delegate) { + return new Wrapped(delegate); + } + + class Wrapped extends StableBridgeAPI.Proxy implements IngestPluginBridge, Closeable { + + private Wrapped(final IngestPlugin delegate) { + super(delegate); + } + + public Map getProcessors(final ProcessorBridge.Parameters parameters) { + return StableBridgeAPI.wrap(this.delegate.getProcessors(parameters.unwrap()), ProcessorBridge.Factory::wrap); + } + + @Override + public IngestPlugin unwrap() { + return this.delegate; + } + + @Override + public void close() throws IOException { + if (this.delegate instanceof Closeable closeableDelegate) { + closeableDelegate.close(); + } + } + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/MetadataBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/MetadataBridge.java new file mode 100644 index 0000000000000..4f0a712ca3505 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/MetadataBridge.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.script; + +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.script.Metadata; + +import java.time.ZonedDateTime; + +public class MetadataBridge extends StableBridgeAPI.Proxy { + public MetadataBridge(final Metadata delegate) { + super(delegate); + } + + public String getIndex() { + return delegate.getIndex(); + } + + public void setIndex(final String index) { + delegate.setIndex(index); + } + + public String getId() { + return delegate.getId(); + } + + public void setId(final String id) { + delegate.setId(id); + } + + public long getVersion() { + return delegate.getVersion(); + } + + public void setVersion(final long version) { + delegate.setVersion(version); + } + + public String getVersionType() { + return delegate.getVersionType(); + } + + public void setVersionType(final String versionType) { + delegate.setVersionType(versionType); + } + + public String getRouting() { + return delegate.getRouting(); + } + + public void setRouting(final String routing) { + delegate.setRouting(routing); + } + + public ZonedDateTime getNow() { + return delegate.getNow(); + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java new file mode 100644 index 0000000000000..ec5af0f7020ac --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/ScriptServiceBridge.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.script; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.logstashbridge.common.SettingsBridge; +import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.painless.PainlessScriptEngine; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.script.IngestConditionalScript; +import org.elasticsearch.script.IngestScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.mustache.MustacheScriptEngine; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.LongSupplier; + +public class ScriptServiceBridge extends StableBridgeAPI.Proxy implements Closeable { + public ScriptServiceBridge wrap(final ScriptService delegate) { + return new ScriptServiceBridge(delegate); + } + + public ScriptServiceBridge(final SettingsBridge settingsBridge, final LongSupplier timeProvider) { + super(getScriptService(settingsBridge.unwrap(), timeProvider)); + } + + public ScriptServiceBridge(ScriptService delegate) { + super(delegate); + } + + private static ScriptService getScriptService(final Settings settings, final LongSupplier timeProvider) { + final List painlessBaseWhitelist = getPainlessBaseWhiteList(); + final Map, List> scriptContexts = Map.of( + IngestScript.CONTEXT, + painlessBaseWhitelist, + IngestConditionalScript.CONTEXT, + painlessBaseWhitelist + ); + final Map scriptEngines = Map.of( + PainlessScriptEngine.NAME, + new PainlessScriptEngine(settings, scriptContexts), + MustacheScriptEngine.NAME, + new MustacheScriptEngine() + ); + return new ScriptService(settings, scriptEngines, ScriptModule.CORE_CONTEXTS, timeProvider); + } + + private static List getPainlessBaseWhiteList() { + return PainlessPlugin.baseWhiteList(); + } + + @Override + public void close() throws IOException { + this.delegate.close(); + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/TemplateScriptBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/TemplateScriptBridge.java new file mode 100644 index 0000000000000..715b357a4ee70 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/script/TemplateScriptBridge.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.script; + +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.script.TemplateScript; + +public class TemplateScriptBridge { + public static class Factory extends StableBridgeAPI.Proxy { + public static Factory wrap(final TemplateScript.Factory delegate) { + return new Factory(delegate); + } + + public Factory(final TemplateScript.Factory delegate) { + super(delegate); + } + + @Override + public TemplateScript.Factory unwrap() { + return this.delegate; + } + } +} diff --git a/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java new file mode 100644 index 0000000000000..13218a9b206a5 --- /dev/null +++ b/libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/threadpool/ThreadPoolBridge.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.logstashbridge.threadpool; + +import org.elasticsearch.logstashbridge.StableBridgeAPI; +import org.elasticsearch.logstashbridge.common.SettingsBridge; +import org.elasticsearch.telemetry.metric.MeterRegistry; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.concurrent.TimeUnit; + +public class ThreadPoolBridge extends StableBridgeAPI.Proxy { + + public ThreadPoolBridge(final SettingsBridge settingsBridge) { + this(new ThreadPool(settingsBridge.unwrap(), MeterRegistry.NOOP)); + } + + public ThreadPoolBridge(final ThreadPool delegate) { + super(delegate); + } + + public static boolean terminate(final ThreadPoolBridge pool, final long timeout, final TimeUnit timeUnit) { + return ThreadPool.terminate(pool.unwrap(), timeout, timeUnit); + } + + public long relativeTimeInMillis() { + return delegate.relativeTimeInMillis(); + } + + public long absoluteTimeInMillis() { + return delegate.absoluteTimeInMillis(); + } +} diff --git a/libs/native/src/main/java/module-info.java b/libs/native/src/main/java/module-info.java index e181398222474..d895df1be1c56 100644 --- a/libs/native/src/main/java/module-info.java +++ b/libs/native/src/main/java/module-info.java @@ -19,8 +19,8 @@ to org.elasticsearch.nativeaccess.jna, org.elasticsearch.server, - org.elasticsearch.systemd, - org.elasticsearch.vec; + org.elasticsearch.simdvec, + org.elasticsearch.systemd; // allows jna to implement a library provider, and ProviderLocator to load it exports org.elasticsearch.nativeaccess.lib to org.elasticsearch.nativeaccess.jna, org.elasticsearch.base; diff --git a/libs/vec/build.gradle b/libs/simdvec/build.gradle similarity index 100% rename from libs/vec/build.gradle rename to libs/simdvec/build.gradle diff --git a/libs/vec/includes.txt b/libs/simdvec/includes.txt similarity index 100% rename from libs/vec/includes.txt rename to libs/simdvec/includes.txt diff --git a/libs/vec/licenses/lucene-core-LICENSE.txt b/libs/simdvec/licenses/lucene-core-LICENSE.txt similarity index 100% rename from libs/vec/licenses/lucene-core-LICENSE.txt rename to libs/simdvec/licenses/lucene-core-LICENSE.txt diff --git a/libs/vec/licenses/lucene-core-NOTICE.txt b/libs/simdvec/licenses/lucene-core-NOTICE.txt similarity index 100% rename from libs/vec/licenses/lucene-core-NOTICE.txt rename to libs/simdvec/licenses/lucene-core-NOTICE.txt diff --git a/libs/vec/native/Dockerfile b/libs/simdvec/native/Dockerfile similarity index 100% rename from libs/vec/native/Dockerfile rename to libs/simdvec/native/Dockerfile diff --git a/libs/vec/native/build.gradle b/libs/simdvec/native/build.gradle similarity index 100% rename from libs/vec/native/build.gradle rename to libs/simdvec/native/build.gradle diff --git a/libs/vec/native/gradle/wrapper/gradle-wrapper.jar b/libs/simdvec/native/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from libs/vec/native/gradle/wrapper/gradle-wrapper.jar rename to libs/simdvec/native/gradle/wrapper/gradle-wrapper.jar diff --git a/libs/vec/native/gradle/wrapper/gradle-wrapper.properties b/libs/simdvec/native/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from libs/vec/native/gradle/wrapper/gradle-wrapper.properties rename to libs/simdvec/native/gradle/wrapper/gradle-wrapper.properties diff --git a/libs/vec/native/gradlew b/libs/simdvec/native/gradlew similarity index 100% rename from libs/vec/native/gradlew rename to libs/simdvec/native/gradlew diff --git a/libs/vec/native/gradlew.bat b/libs/simdvec/native/gradlew.bat similarity index 100% rename from libs/vec/native/gradlew.bat rename to libs/simdvec/native/gradlew.bat diff --git a/libs/vec/native/publish_vec_binaries.sh b/libs/simdvec/native/publish_vec_binaries.sh similarity index 100% rename from libs/vec/native/publish_vec_binaries.sh rename to libs/simdvec/native/publish_vec_binaries.sh diff --git a/libs/vec/native/settings.gradle b/libs/simdvec/native/settings.gradle similarity index 100% rename from libs/vec/native/settings.gradle rename to libs/simdvec/native/settings.gradle diff --git a/libs/vec/native/src/vec/c/aarch64/vec.c b/libs/simdvec/native/src/vec/c/aarch64/vec.c similarity index 100% rename from libs/vec/native/src/vec/c/aarch64/vec.c rename to libs/simdvec/native/src/vec/c/aarch64/vec.c diff --git a/libs/vec/native/src/vec/c/amd64/vec.c b/libs/simdvec/native/src/vec/c/amd64/vec.c similarity index 100% rename from libs/vec/native/src/vec/c/amd64/vec.c rename to libs/simdvec/native/src/vec/c/amd64/vec.c diff --git a/libs/vec/native/src/vec/headers/vec.h b/libs/simdvec/native/src/vec/headers/vec.h similarity index 100% rename from libs/vec/native/src/vec/headers/vec.h rename to libs/simdvec/native/src/vec/headers/vec.h diff --git a/libs/vec/src/main/java/module-info.java b/libs/simdvec/src/main/java/module-info.java similarity index 81% rename from libs/vec/src/main/java/module-info.java rename to libs/simdvec/src/main/java/module-info.java index a8a7c7982fbe0..05a2e24d29fca 100644 --- a/libs/vec/src/main/java/module-info.java +++ b/libs/simdvec/src/main/java/module-info.java @@ -6,9 +6,9 @@ * Side Public License, v 1. */ -module org.elasticsearch.vec { +module org.elasticsearch.simdvec { requires org.elasticsearch.nativeaccess; requires org.apache.lucene.core; - exports org.elasticsearch.vec to org.elasticsearch.server; + exports org.elasticsearch.simdvec to org.elasticsearch.server; } diff --git a/libs/vec/src/main/java/org/elasticsearch/vec/VectorScorerFactory.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java similarity index 98% rename from libs/vec/src/main/java/org/elasticsearch/vec/VectorScorerFactory.java rename to libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java index 6005572786817..88c4a59d0ffdb 100644 --- a/libs/vec/src/main/java/org/elasticsearch/vec/VectorScorerFactory.java +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactory.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec; +package org.elasticsearch.simdvec; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.IndexInput; diff --git a/libs/vec/src/main/java/org/elasticsearch/vec/VectorScorerFactoryImpl.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java similarity index 97% rename from libs/vec/src/main/java/org/elasticsearch/vec/VectorScorerFactoryImpl.java rename to libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java index 3d6d0db387186..b5f5d1ef5c67d 100644 --- a/libs/vec/src/main/java/org/elasticsearch/vec/VectorScorerFactoryImpl.java +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec; +package org.elasticsearch.simdvec; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.IndexInput; diff --git a/libs/vec/src/main/java/org/elasticsearch/vec/VectorSimilarityType.java b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorSimilarityType.java similarity index 97% rename from libs/vec/src/main/java/org/elasticsearch/vec/VectorSimilarityType.java rename to libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorSimilarityType.java index 68f14e9b72623..0e321771353a3 100644 --- a/libs/vec/src/main/java/org/elasticsearch/vec/VectorSimilarityType.java +++ b/libs/simdvec/src/main/java/org/elasticsearch/simdvec/VectorSimilarityType.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec; +package org.elasticsearch.simdvec; import org.apache.lucene.index.VectorSimilarityFunction; diff --git a/libs/vec/src/main21/java/org/elasticsearch/vec/VectorScorerFactoryImpl.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java similarity index 87% rename from libs/vec/src/main21/java/org/elasticsearch/vec/VectorScorerFactoryImpl.java rename to libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java index 944e886041b8c..7c120d53a28ff 100644 --- a/libs/vec/src/main21/java/org/elasticsearch/vec/VectorScorerFactoryImpl.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/VectorScorerFactoryImpl.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec; +package org.elasticsearch.simdvec; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.store.FilterIndexInput; @@ -16,10 +16,10 @@ import org.apache.lucene.util.hnsw.RandomVectorScorerSupplier; import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues; import org.elasticsearch.nativeaccess.NativeAccess; -import org.elasticsearch.vec.internal.Int7SQVectorScorer; -import org.elasticsearch.vec.internal.Int7SQVectorScorerSupplier.DotProductSupplier; -import org.elasticsearch.vec.internal.Int7SQVectorScorerSupplier.EuclideanSupplier; -import org.elasticsearch.vec.internal.Int7SQVectorScorerSupplier.MaxInnerProductSupplier; +import org.elasticsearch.simdvec.internal.Int7SQVectorScorer; +import org.elasticsearch.simdvec.internal.Int7SQVectorScorerSupplier.DotProductSupplier; +import org.elasticsearch.simdvec.internal.Int7SQVectorScorerSupplier.EuclideanSupplier; +import org.elasticsearch.simdvec.internal.Int7SQVectorScorerSupplier.MaxInnerProductSupplier; import java.util.Optional; diff --git a/libs/vec/src/main21/java/org/elasticsearch/vec/internal/Int7SQVectorScorer.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java similarity index 95% rename from libs/vec/src/main21/java/org/elasticsearch/vec/internal/Int7SQVectorScorer.java rename to libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java index 95bf79eb96609..bdb4f22b3ade2 100644 --- a/libs/vec/src/main21/java/org/elasticsearch/vec/internal/Int7SQVectorScorer.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec.internal; +package org.elasticsearch.simdvec.internal; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.util.hnsw.RandomVectorScorer; diff --git a/libs/vec/src/main21/java/org/elasticsearch/vec/internal/Int7SQVectorScorerSupplier.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java similarity index 99% rename from libs/vec/src/main21/java/org/elasticsearch/vec/internal/Int7SQVectorScorerSupplier.java rename to libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java index cb20fcd3c39b3..b1410b03cd8ce 100644 --- a/libs/vec/src/main21/java/org/elasticsearch/vec/internal/Int7SQVectorScorerSupplier.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorerSupplier.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec.internal; +package org.elasticsearch.simdvec.internal; import org.apache.lucene.store.MemorySegmentAccessInput; import org.apache.lucene.util.hnsw.RandomVectorScorer; diff --git a/libs/vec/src/main21/java/org/elasticsearch/vec/internal/Similarities.java b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Similarities.java similarity index 97% rename from libs/vec/src/main21/java/org/elasticsearch/vec/internal/Similarities.java rename to libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Similarities.java index 8f78e5a385a12..eea319541437b 100644 --- a/libs/vec/src/main21/java/org/elasticsearch/vec/internal/Similarities.java +++ b/libs/simdvec/src/main21/java/org/elasticsearch/simdvec/internal/Similarities.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec.internal; +package org.elasticsearch.simdvec.internal; import org.elasticsearch.nativeaccess.NativeAccess; import org.elasticsearch.nativeaccess.VectorSimilarityFunctions; diff --git a/libs/vec/src/main22/java/org/elasticsearch/vec/internal/Int7SQVectorScorer.java b/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java similarity index 97% rename from libs/vec/src/main22/java/org/elasticsearch/vec/internal/Int7SQVectorScorer.java rename to libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java index b835e7734a481..90a7e5a23dd4c 100644 --- a/libs/vec/src/main22/java/org/elasticsearch/vec/internal/Int7SQVectorScorer.java +++ b/libs/simdvec/src/main22/java/org/elasticsearch/simdvec/internal/Int7SQVectorScorer.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec.internal; +package org.elasticsearch.simdvec.internal; import org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer; import org.apache.lucene.index.VectorSimilarityFunction; @@ -21,8 +21,8 @@ import java.lang.foreign.MemorySegment; import java.util.Optional; -import static org.elasticsearch.vec.internal.Similarities.dotProduct7u; -import static org.elasticsearch.vec.internal.Similarities.squareDistance7u; +import static org.elasticsearch.simdvec.internal.Similarities.dotProduct7u; +import static org.elasticsearch.simdvec.internal.Similarities.squareDistance7u; public abstract sealed class Int7SQVectorScorer extends RandomVectorScorer.AbstractRandomVectorScorer { diff --git a/libs/vec/src/test/java/org/elasticsearch/vec/AbstractVectorTestCase.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/AbstractVectorTestCase.java similarity index 98% rename from libs/vec/src/test/java/org/elasticsearch/vec/AbstractVectorTestCase.java rename to libs/simdvec/src/test/java/org/elasticsearch/simdvec/AbstractVectorTestCase.java index 7d3b85cb6c54a..1734bef80389d 100644 --- a/libs/vec/src/test/java/org/elasticsearch/vec/AbstractVectorTestCase.java +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/AbstractVectorTestCase.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec; +package org.elasticsearch.simdvec; import org.apache.lucene.util.quantization.ScalarQuantizedVectorSimilarity; import org.elasticsearch.test.ESTestCase; diff --git a/libs/vec/src/test/java/org/elasticsearch/vec/VectorScorerFactoryTests.java b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java similarity index 98% rename from libs/vec/src/test/java/org/elasticsearch/vec/VectorScorerFactoryTests.java rename to libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java index dfd26e887b2ba..93c6da73f4179 100644 --- a/libs/vec/src/test/java/org/elasticsearch/vec/VectorScorerFactoryTests.java +++ b/libs/simdvec/src/test/java/org/elasticsearch/simdvec/VectorScorerFactoryTests.java @@ -6,7 +6,7 @@ * Side Public License, v 1. */ -package org.elasticsearch.vec; +package org.elasticsearch.simdvec; import com.carrotsearch.randomizedtesting.generators.RandomNumbers; @@ -38,11 +38,11 @@ import java.util.stream.IntStream; import static org.apache.lucene.codecs.hnsw.ScalarQuantizedVectorScorer.quantizeQuery; +import static org.elasticsearch.simdvec.VectorSimilarityType.COSINE; +import static org.elasticsearch.simdvec.VectorSimilarityType.DOT_PRODUCT; +import static org.elasticsearch.simdvec.VectorSimilarityType.EUCLIDEAN; +import static org.elasticsearch.simdvec.VectorSimilarityType.MAXIMUM_INNER_PRODUCT; import static org.elasticsearch.test.hamcrest.OptionalMatchers.isEmpty; -import static org.elasticsearch.vec.VectorSimilarityType.COSINE; -import static org.elasticsearch.vec.VectorSimilarityType.DOT_PRODUCT; -import static org.elasticsearch.vec.VectorSimilarityType.EUCLIDEAN; -import static org.elasticsearch.vec.VectorSimilarityType.MAXIMUM_INNER_PRODUCT; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/CopyingXContentParser.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/CopyingXContentParser.java new file mode 100644 index 0000000000000..b8e6e1330e0c2 --- /dev/null +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/CopyingXContentParser.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.xcontent; + +import java.io.IOException; + +/** + * A parser that copies data that was parsed into a {@link XContentBuilder}. + * This parser naturally has some memory and runtime overhead to perform said copying. + * Use with {@link XContentSubParser} to preserve the entire object. + */ +public class CopyingXContentParser extends FilterXContentParserWrapper { + private final XContentBuilder builder; + + public CopyingXContentParser(XContentParser delegate) throws IOException { + super(delegate); + this.builder = XContentBuilder.builder(delegate.contentType().xContent()); + switch (delegate.currentToken()) { + case START_OBJECT -> builder.startObject(); + case START_ARRAY -> builder.startArray(); + default -> throw new IllegalArgumentException( + "can only copy parsers pointed to START_OBJECT or START_ARRAY but found: " + delegate.currentToken() + ); + } + } + + @Override + public Token nextToken() throws IOException { + XContentParser.Token next = delegate().nextToken(); + builder.copyCurrentEvent(delegate()); + return next; + } + + public XContentBuilder getBuilder() { + return builder; + } +} diff --git a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java index 2143814565a51..1be4594b097a6 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java +++ b/libs/x-content/src/main/java/org/elasticsearch/xcontent/XContentBuilder.java @@ -1220,6 +1220,18 @@ public XContentBuilder rawValue(String value) throws IOException { return this; } + /** + * Copies current event from parser into this builder. + * The difference with {@link XContentBuilder#copyCurrentStructure(XContentParser)} + * is that this method does not copy sub-objects as a single entity. + * @param parser + * @throws IOException + */ + public XContentBuilder copyCurrentEvent(XContentParser parser) throws IOException { + generator.copyCurrentEvent(parser); + return this; + } + public XContentBuilder copyCurrentStructure(XContentParser parser) throws IOException { generator.copyCurrentStructure(parser); return this; diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index 1bd4d54b9c804..369f3a9d42724 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse; import org.elasticsearch.action.admin.indices.close.CloseIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverAction; import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction; @@ -127,6 +128,16 @@ public void setup() throws Exception { response = client.execute(CreateDataStreamAction.INSTANCE, request).get(); assertTrue(response.isAcknowledged()); + // Initialize the failure store. + RolloverRequest rolloverRequest = new RolloverRequest("with-fs", null); + rolloverRequest.setIndicesOptions( + IndicesOptions.builder(rolloverRequest.indicesOptions()) + .failureStoreOptions(b -> b.includeRegularIndices(false).includeFailureIndices(true)) + .build() + ); + response = client.execute(RolloverAction.INSTANCE, rolloverRequest).get(); + assertTrue(response.isAcknowledged()); + // Resolve backing index names after data streams have been created: // (these names have a date component, and running around midnight could lead to test failures otherwise) GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { "*" }); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 8a343ff9cf853..f95d9a0b0431f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -62,7 +62,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword" }, "pid": { @@ -86,7 +86,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword", "time_series_dimension": "true" }, @@ -110,7 +110,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { private static final String LOG_DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "message": "%s", @@ -121,7 +121,7 @@ public class LogsDataStreamIT extends ESSingleNodeTestCase { private static final String TIME_SERIES_DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "ip_address": "%s", @@ -207,7 +207,7 @@ public void testIndexModeLogsAndTimeSeriesSwitching() throws IOException, Execut final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); - final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "hostname"); + final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "host.name"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); createDataStream(client(), dataStreamName); @@ -224,7 +224,7 @@ public void testIndexModeLogsAndTimeSeriesSwitching() throws IOException, Execut assertDataStreamBackingIndicesModes(dataStreamName, List.of(IndexMode.LOGS, IndexMode.TIME_SERIES, IndexMode.LOGS)); } - public void testInvalidIndexModeTimeSeriesSwitchWithoutROutingPath() throws IOException, ExecutionException, InterruptedException { + public void testInvalidIndexModeTimeSeriesSwitchWithoutRoutingPath() throws IOException, ExecutionException, InterruptedException { final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); @@ -250,7 +250,7 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc final String dataStreamName = generateDataStreamName("custom"); final List indexPatterns = List.of("custom-*-*"); final Map logsSettings = Map.of("index.mode", "logs"); - final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "hostname"); + final Map timeSeriesSettings = Map.of("index.mode", "time_series", "index.routing_path", "host.name"); putComposableIndexTemplate(client(), "custom-composable-template", LOGS_OR_STANDARD_MAPPING, logsSettings, indexPatterns); createDataStream(client(), dataStreamName); @@ -269,8 +269,9 @@ public void testInvalidIndexModeTimeSeriesSwitchWithoutDimensions() throws IOExc assertThat( exception.getCause().getCause().getMessage(), Matchers.equalTo( - "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields with " - + "a list of dimensions in [time_series_dimensions] and without the [script] parameter. [hostname] was not a dimension." + "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields " + + "with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [host.name] was not a " + + "dimension." ) ); } diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java index 27cd5697fd0f7..4af3a3844e453 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/FailureStoreQueryParamIT.java @@ -51,6 +51,8 @@ public void setup() throws IOException { assertOK(client().performRequest(putComposableIndexTemplateRequest)); assertOK(client().performRequest(new Request("PUT", "/_data_stream/" + DATA_STREAM_NAME))); + // Initialize the failure store. + assertOK(client().performRequest(new Request("POST", DATA_STREAM_NAME + "/_rollover?target_failure_store"))); ensureGreen(DATA_STREAM_NAME); final Response dataStreamResponse = client().performRequest(new Request("GET", "/_data_stream/" + DATA_STREAM_NAME)); diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java index c18bcf750242f..d3ec5b29ff5b9 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamRestIT.java @@ -80,7 +80,7 @@ private static void waitForLogs(RestClient client) throws Exception { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword" }, "pid": { @@ -116,7 +116,7 @@ private static void waitForLogs(RestClient client) throws Exception { "@timestamp" : { "type": "date" }, - "hostname": { + "host.name": { "type": "keyword", "time_series_dimension": "true" }, @@ -138,7 +138,7 @@ private static void waitForLogs(RestClient client) throws Exception { private static final String DOC_TEMPLATE = """ { "@timestamp": "%s", - "hostname": "%s", + "host.name": "%s", "pid": "%d", "method": "%s", "message": "%s", diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java new file mode 100644 index 0000000000000..dcd2457b88f18 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeDisabledRestTestIT.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class LogsIndexModeDisabledRestTestIT extends LogsIndexModeRestTestIT { + + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + public void testLogsSettingsIndexModeDisabled() throws IOException { + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + assertThat(indexMode, Matchers.not(equalTo(IndexMode.LOGS.getName()))); + } + +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java new file mode 100644 index 0000000000000..832267cebf97c --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeEnabledRestTestIT.java @@ -0,0 +1,226 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.cluster.local.distribution.DistributionType; +import org.hamcrest.Matchers; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class LogsIndexModeEnabledRestTestIT extends LogsIndexModeRestTestIT { + + @ClassRule() + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .distribution(DistributionType.DEFAULT) + .module("constant-keyword") + .module("data-streams") + .module("mapper-extras") + .module("x-pack-aggregate-metric") + .module("x-pack-stack") + .setting("xpack.security.enabled", "false") + .setting("xpack.license.self_generated.type", "trial") + .setting("cluster.logsdb.enabled", "true") + .build(); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + @Before + public void setup() throws Exception { + client = client(); + waitForLogs(client); + } + + private RestClient client; + + private static final String MAPPINGS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + } + } + } + } + }"""; + + private static final String ALTERNATE_HOST_MAPPING = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host.cloud_region": { + "type": "keyword" + }, + "host.availability_zone": { + "type": "keyword" + } + } + } + } + }"""; + + private static final String HOST_MAPPING_AS_OBJECT_DEFAULT_SUBOBJECTS = """ + { + "template": { + "mappings": { + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host": { + "type": "object", + "properties": { + "cloud_region": { + "type": "keyword" + }, + "availability_zone": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }"""; + + private static final String HOST_MAPPING_AS_OBJECT_NON_DEFAULT_SUBOBJECTS = """ + { + "template": { + "mappings": { + "dynamic": "strict", + "properties": { + "method": { + "type": "keyword" + }, + "message": { + "type": "text" + }, + "host": { + "type": "object", + "subobjects": false, + "properties": { + "cloud_region": { + "type": "keyword" + }, + "availability_zone": { + "type": "keyword" + }, + "name": { + "type": "keyword" + } + } + } + } + } + } + }"""; + + private static String BULK_INDEX_REQUEST = """ + { "create": {}} + { "@timestamp": "2023-01-01T05:11:00Z", "host.name": "foo", "method" : "PUT", "message": "foo put message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "bar", "method" : "POST", "message": "bar post message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } + { "create": {}} + { "@timestamp": "2023-01-01T05:13:00Z", "host.name": "baz", "method" : "PUT", "message": "baz put message" } + """; + + private static String BULK_INDEX_REQUEST_WITH_HOST = """ + { "create": {}} + { "@timestamp": "2023-01-01T05:11:00Z", "method" : "PUT", "message": "foo put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "ahdta-876584" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "method" : "POST", "message": "bar post message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "tyrou-447898" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:12:00Z", "method" : "PUT", "message": "baz put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4a", "name" : "uuopl-162899" } } + { "create": {}} + { "@timestamp": "2023-01-01T05:13:00Z", "method" : "PUT", "message": "baz put message", \ + "host": { "cloud_region" : "us-west", "availability_zone" : "us-west-4b", "name" : "fdfgf-881197" } } + """; + + public void testCreateDataStream() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String indexMode = (String) getSetting(client, getDataStreamBackingIndex(client, "logs-custom-dev", 0), "index.mode"); + assertThat(indexMode, equalTo(IndexMode.LOGS.getName())); + } + + public void testBulkIndexing() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithFlatHostProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", ALTERNATE_HOST_MAPPING)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithObjectHostDefaultSubobjectsProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", HOST_MAPPING_AS_OBJECT_DEFAULT_SUBOBJECTS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testBulkIndexingWithObjectHostSubobjectsFalseProperties() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", HOST_MAPPING_AS_OBJECT_NON_DEFAULT_SUBOBJECTS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final Response response = bulkIndex(client, "logs-custom-dev", () -> BULK_INDEX_REQUEST_WITH_HOST); + assertOK(response); + assertThat(entityAsMap(response).get("errors"), Matchers.equalTo(false)); + } + + public void testRolloverDataStream() throws IOException { + assertOK(putComponentTemplate(client, "logs@custom", MAPPINGS)); + assertOK(createDataStream(client, "logs-custom-dev")); + final String firstBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 0); + assertOK(rolloverDataStream(client, "logs-custom-dev")); + final String secondBackingIndex = getDataStreamBackingIndex(client, "logs-custom-dev", 1); + assertThat(firstBackingIndex, Matchers.not(equalTo(secondBackingIndex))); + assertThat(getDataStreamBackingIndices(client, "logs-custom-dev").size(), equalTo(2)); + } +} diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java new file mode 100644 index 0000000000000..ff45096146280 --- /dev/null +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/LogsIndexModeRestTestIT.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.datastreams.logsdb; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +public abstract class LogsIndexModeRestTestIT extends ESRestTestCase { + protected static void waitForLogs(RestClient client) throws Exception { + assertBusy(() -> { + try { + final Request request = new Request("GET", "_index_template/logs"); + assertOK(client.performRequest(request)); + } catch (ResponseException e) { + fail(e.getMessage()); + } + }); + } + + protected static Response putComponentTemplate(final RestClient client, final String templateName, final String mappings) + throws IOException { + final Request request = new Request("PUT", "/_component_template/" + templateName); + request.setJsonEntity(mappings); + return client.performRequest(request); + } + + protected static Response createDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("PUT", "_data_stream/" + dataStreamName)); + } + + protected static Response rolloverDataStream(final RestClient client, final String dataStreamName) throws IOException { + return client.performRequest(new Request("POST", "/" + dataStreamName + "/_rollover")); + } + + @SuppressWarnings("unchecked") + protected static String getDataStreamBackingIndex(final RestClient client, final String dataStreamName, int backingIndex) + throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.get(backingIndex).get("index_name"); + } + + @SuppressWarnings("unchecked") + protected static List getDataStreamBackingIndices(final RestClient client, final String dataStreamName) throws IOException { + final Request request = new Request("GET", "_data_stream/" + dataStreamName); + final List dataStreams = (List) entityAsMap(client.performRequest(request)).get("data_streams"); + final Map dataStream = (Map) dataStreams.get(0); + final List> backingIndices = (List>) dataStream.get("indices"); + return backingIndices.stream().map(map -> map.get("indices")).collect(Collectors.toList()); + } + + @SuppressWarnings("unchecked") + protected static Object getSetting(final RestClient client, final String indexName, final String setting) throws IOException { + final Request request = new Request("GET", "/" + indexName + "/_settings?flat_settings=true&include_defaults=true"); + final Map settings = ((Map>) entityAsMap(client.performRequest(request)).get(indexName)) + .get("settings"); + + return settings.get(setting); + } + + protected static Response bulkIndex(final RestClient client, final String dataStreamName, final Supplier bulkSupplier) + throws IOException { + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + bulkRequest.setJsonEntity(bulkSupplier.get()); + bulkRequest.addParameter("refresh", "true"); + return client.performRequest(bulkRequest); + } +} diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java index 8ca5fe1fcdbcf..6a1f8031ce7c6 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java @@ -315,7 +315,7 @@ private ClusterState createDataStream(ClusterState state, String name, Instant t TimeValue.ZERO, false ); - return createDataStreamService.createDataStream(request, state, ActionListener.noop()); + return createDataStreamService.createDataStream(request, state, ActionListener.noop(), false); } private MetadataRolloverService.RolloverResult rolloverOver(ClusterState state, String name, Instant time) throws Exception { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml index 5b88f414634b5..609b0c3d0c33c 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/10_basic.yml @@ -261,6 +261,12 @@ setup: index: default_pipeline: "data_stream_pipeline" final_pipeline: "data_stream_final_pipeline" + mappings: + properties: + '@timestamp': + type: date + count: + type: long - do: indices.create_data_stream: @@ -272,6 +278,23 @@ setup: name: failure-data-stream2 - is_true: acknowledged + # Initialize failure store + - do: + index: + index: failure-data-stream1 + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + # Initialize failure store + - do: + index: + index: failure-data-stream2 + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + - do: cluster.health: wait_for_status: green @@ -281,7 +304,7 @@ setup: name: "*" - match: { data_streams.0.name: failure-data-stream1 } - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 1 } + - match: { data_streams.0.generation: 2 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - match: { data_streams.0.status: 'GREEN' } @@ -289,18 +312,18 @@ setup: - match: { data_streams.0.hidden: false } - match: { data_streams.0.failure_store.enabled: true } - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/'} + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000002/'} - match: { data_streams.1.name: failure-data-stream2 } - match: { data_streams.1.timestamp_field.name: '@timestamp' } - - match: { data_streams.1.generation: 1 } + - match: { data_streams.1.generation: 2 } - length: { data_streams.1.indices: 1 } - match: { data_streams.1.indices.0.index_name: '/\.ds-failure-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - match: { data_streams.1.template: 'my-template4' } - match: { data_streams.1.hidden: false } - match: { data_streams.1.failure_store.enabled: true } - length: { data_streams.1.failure_store.indices: 1 } - - match: { data_streams.1.failure_store.indices.0.index_name: '/\.fs-failure-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.1.failure_store.indices.0.index_name: '/\.fs-failure-data-stream2-(\d{4}\.\d{2}\.\d{2}-)?000002/' } # save the backing index names for later use - set: { data_streams.0.indices.0.index_name: idx0name } @@ -603,7 +626,7 @@ setup: index: $idx0name --- -"Delete data stream with failure stores": +"Delete data stream with failure store": - requires: cluster_features: ["gte_v8.15.0"] reason: "data stream failure stores REST structure changed in 8.15+" @@ -617,12 +640,28 @@ setup: index_patterns: [ failure-data-stream1 ] data_stream: failure_store: true + template: + mappings: + properties: + '@timestamp': + type: date + count: + type: long - do: indices.create_data_stream: name: failure-data-stream1 - is_true: acknowledged + # Initialize failure store + - do: + index: + index: failure-data-stream1 + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + - do: indices.create: index: test_index @@ -650,11 +689,11 @@ setup: indices.get_data_stream: {} - match: { data_streams.0.name: failure-data-stream1 } - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 1 } + - match: { data_streams.0.generation: 2 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - do: indices.delete_data_stream: @@ -676,6 +715,74 @@ setup: name: my-template4 - is_true: acknowledged +--- +"Delete data stream with failure store uninitialized": + - requires: + cluster_features: ["gte_v8.15.0"] + reason: "data stream failure stores REST structure changed in 8.15+" + + - do: + allowed_warnings: + - "index template [my-template4] has index patterns [failure-data-stream1] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template4] will take precedence during new index creation" + indices.put_index_template: + name: my-template4 + body: + index_patterns: [ failure-data-stream1 ] + data_stream: + failure_store: true + + - do: + indices.create_data_stream: + name: failure-data-stream1 + - is_true: acknowledged + + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + number_of_replicas: 1 + + # save the backing index names for later use + - do: + indices.get_data_stream: + name: failure-data-stream1 + + - set: { data_streams.0.indices.0.index_name: idx0name } + - length: { data_streams.0.failure_store.indices: 0 } + + - do: + indices.get: + index: ['.ds-failure-data-stream1-*000001', 'test_index'] + + - is_true: test_index.settings + - is_true: .$idx0name.settings + + - do: + indices.get_data_stream: {} + - match: { data_streams.0.name: failure-data-stream1 } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-failure-data-stream1-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 0 } + + - do: + indices.delete_data_stream: + name: failure-data-stream1 + - is_true: acknowledged + + - do: + catch: missing + indices.get: + index: $idx0name + + - do: + indices.delete_index_template: + name: my-template4 + - is_true: acknowledged + --- "Delete data stream missing behaviour": - requires: diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml index a3baa524259b8..3c6d29d939226 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/170_modify_data_stream.yml @@ -105,6 +105,13 @@ index_patterns: [data-*] data_stream: failure_store: true + template: + mappings: + properties: + '@timestamp': + type: date + count: + type: long - do: indices.create_data_stream: @@ -116,6 +123,23 @@ name: data-stream-for-modification2 - is_true: acknowledged + # Initialize failure store + - do: + index: + index: data-stream-for-modification + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + # Initialize failure store + - do: + index: + index: data-stream-for-modification2 + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + # rollover data stream to create new failure store index - do: indices.rollover: @@ -168,7 +192,7 @@ name: "data-stream-for-modification" - match: { data_streams.0.name: data-stream-for-modification } - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 3 } + - match: { data_streams.0.generation: 4 } - length: { data_streams.0.indices: 1 } - length: { data_streams.0.failure_store.indices: 3 } - match: { data_streams.0.indices.0.index_name: $write_index } @@ -187,17 +211,6 @@ index: test_index2 failure_store: true - # We are not allowed to remove the write index for the failure store - - do: - catch: /cannot remove backing index \[.*\] of data stream \[data-stream-for-modification\] because it is the write index/ - indices.modify_data_stream: - body: - actions: - - remove_backing_index: - data_stream: "data-stream-for-modification" - index: $write_failure_index - failure_store: true - # We will not accept an index that is already part of the data stream's backing indices - do: catch: /cannot add index \[.*\] to data stream \[data-stream-for-modification\] because it is already a backing index on data stream \[data-stream-for-modification\]/ @@ -267,13 +280,112 @@ name: "data-stream-for-modification" - match: { data_streams.0.name: data-stream-for-modification } - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 4 } + - match: { data_streams.0.generation: 5 } - length: { data_streams.0.indices: 1 } - length: { data_streams.0.failure_store.indices: 2 } - match: { data_streams.0.indices.0.index_name: $write_index } - match: { data_streams.0.failure_store.indices.0.index_name: $first_failure_index } - match: { data_streams.0.failure_store.indices.1.index_name: $write_failure_index } + # Remove write index of the failure store + - do: + indices.modify_data_stream: + body: + actions: + - remove_backing_index: + data_stream: "data-stream-for-modification" + index: $write_failure_index + failure_store: true + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "data-stream-for-modification" + - match: { data_streams.0.name: data-stream-for-modification } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 6 } + - length: { data_streams.0.indices: 1 } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.indices.0.index_name: $write_index } + - match: { data_streams.0.failure_store.indices.0.index_name: $first_failure_index } + + # Remove the last write index of the failure store + - do: + indices.modify_data_stream: + body: + actions: + - remove_backing_index: + data_stream: "data-stream-for-modification" + index: $first_failure_index + failure_store: true + - is_true: acknowledged + + - do: + indices.get_data_stream: + name: "data-stream-for-modification" + - match: { data_streams.0.name: data-stream-for-modification } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 7 } + - length: { data_streams.0.indices: 1 } + - length: { data_streams.0.failure_store.indices: 0 } + - match: { data_streams.0.indices.0.index_name: $write_index } + + # Doing these checks again to make sure we still return the same error with an empty failure store + # We will not accept an index that is already part of the data stream's backing indices + - do: + catch: /cannot add index \[.*\] to data stream \[data-stream-for-modification\] because it is already a backing index on data stream \[data-stream-for-modification\]/ + indices.modify_data_stream: + body: + actions: + - add_backing_index: + data_stream: "data-stream-for-modification" + index: $write_index + failure_store: true + + # We will not accept an index that is already part of a different data stream's backing indices + - do: + catch: /cannot add index \[.*\] to data stream \[data-stream-for-modification\] because it is already a backing index on data stream \[data-stream-for-modification2\]/ + indices.modify_data_stream: + body: + actions: + - add_backing_index: + data_stream: "data-stream-for-modification" + index: $second_write_index + failure_store: true + + # We will not accept an index that is already part of a different data stream's failure store + - do: + catch: /cannot add index \[.*\] to data stream \[data-stream-for-modification\] because it is already a failure store index on data stream \[data-stream-for-modification2\]/ + indices.modify_data_stream: + body: + actions: + - add_backing_index: + data_stream: "data-stream-for-modification" + index: $second_write_failure_index + failure_store: true + + # We will return a failed response if we try to remove an index from the failure store that is not present + - do: + catch: /index \[.*\] not found/ + indices.modify_data_stream: + body: + actions: + - remove_backing_index: + data_stream: "data-stream-for-modification" + index: $write_index + failure_store: true + + # Add index to empty failure store + - do: + indices.modify_data_stream: + body: + actions: + - add_backing_index: + data_stream: "data-stream-for-modification" + index: "test_index1" + failure_store: true + - is_true: acknowledged + - do: indices.delete_data_stream: name: data-stream-for-modification diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml index 5682e2235abc8..04c70ee380d4f 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml @@ -181,7 +181,7 @@ teardown: - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - match: { data_streams.0.failure_store.enabled: true } - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - do: search: @@ -193,7 +193,7 @@ teardown: search: index: .fs-logs-foobar-* - length: { hits.hits: 1 } - - match: { hits.hits.0._index: "/\\.fs-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" } + - match: { hits.hits.0._index: "/\\.fs-logs-foobar-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - exists: hits.hits.0._source.@timestamp - not_exists: hits.hits.0._source.count - match: { hits.hits.0._source.document.index: 'logs-foobar' } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml index 8cdfe3d97bbb8..dcbb0d2e465db 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/200_rollover_failure_store.yml @@ -39,14 +39,23 @@ teardown: ignore: 404 --- "Roll over a data stream's failure store without conditions": + # Initialize failure store + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + - do: indices.rollover: alias: "data-stream-for-rollover" target_failure_store: true - match: { acknowledged: true } - - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" } - - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } - match: { rolled_over: true } - match: { dry_run: false } @@ -56,12 +65,12 @@ teardown: - match: { data_streams.0.name: data-stream-for-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 2 } + - match: { data_streams.0.generation: 3 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } --- "Roll over a data stream's failure store with conditions": @@ -82,8 +91,8 @@ teardown: max_docs: 1 - match: { acknowledged: true } - - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000001/" } - - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { old_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } - match: { rolled_over: true } - match: { dry_run: false } @@ -93,22 +102,31 @@ teardown: - match: { data_streams.0.name: data-stream-for-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 2 } + - match: { data_streams.0.generation: 3 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } --- "Don't roll over a data stream's failure store when conditions aren't met": + # Initialize failure store + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + - do: indices.rollover: alias: "data-stream-for-rollover" target_failure_store: true body: conditions: - max_docs: 1 + max_primary_shard_docs: 2 - match: { acknowledged: false } - match: { rolled_over: false } @@ -119,11 +137,11 @@ teardown: name: "*" - match: { data_streams.0.name: data-stream-for-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 1 } + - match: { data_streams.0.generation: 2 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } --- "Lazily roll over a data stream's failure store after a shard failure": @@ -135,6 +153,15 @@ teardown: path: /{index}/_rollover capabilities: [lazy-rollover-failure-store] + # Initialize failure store + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + # Mark the failure store for lazy rollover - do: indices.rollover: @@ -151,11 +178,11 @@ teardown: name: "*" - match: { data_streams.0.name: data-stream-for-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 1 } + - match: { data_streams.0.generation: 2 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - do: index: @@ -171,24 +198,20 @@ teardown: - match: { data_streams.0.name: data-stream-for-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 2 } + - match: { data_streams.0.generation: 3 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } + # Ensure failure got redirected to new index (after rollover). - do: search: index: .fs-data-stream-for-rollover-* - - length: { hits.hits: 1 } + - length: { hits.hits: 2 } - match: { hits.hits.0._index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } - - exists: hits.hits.0._source.@timestamp - - not_exists: hits.hits.0._source.count - - match: { hits.hits.0._source.document.index: 'data-stream-for-rollover' } - - match: { hits.hits.0._source.document.source.@timestamp: '2020-12-12' } - - match: { hits.hits.0._source.document.source.count: 'invalid value' } - - match: { hits.hits.0._source.error.type: 'document_parsing_exception' } + - match: { hits.hits.1._index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } --- "Lazily roll over a data stream's failure store after an ingest failure": @@ -234,6 +257,15 @@ teardown: indices.create_data_stream: name: data-stream-for-lazy-rollover + # Initialize failure store + - do: + index: + index: data-stream-for-lazy-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + # Mark the failure store for lazy rollover - do: indices.rollover: @@ -250,11 +282,11 @@ teardown: name: "*" - match: { data_streams.0.name: data-stream-for-lazy-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 1 } + - match: { data_streams.0.generation: 2 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - do: index: @@ -270,13 +302,20 @@ teardown: - match: { data_streams.0.name: data-stream-for-lazy-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 2 } + - match: { data_streams.0.generation: 3 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 2 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-data-stream-for-lazy-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } + # Ensure failure got redirected to new index (after rollover). + - do: + search: + index: .fs-data-stream-for-lazy-rollover-* + - length: { hits.hits: 2 } + - match: { hits.hits.0._index: "/\\.fs-data-stream-for-lazy-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { hits.hits.1._index: "/\\.fs-data-stream-for-lazy-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } --- "A failure store marked for lazy rollover should only be rolled over when there is a failure": - requires: @@ -287,6 +326,15 @@ teardown: path: /{index}/_rollover capabilities: [lazy-rollover-failure-store] + # Initialize failure store + - do: + index: + index: data-stream-for-rollover + refresh: true + body: + '@timestamp': '2020-12-12' + count: 'invalid value' + # Mark the failure store for lazy rollover - do: indices.rollover: @@ -303,11 +351,11 @@ teardown: name: "*" - match: { data_streams.0.name: data-stream-for-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } - - match: { data_streams.0.generation: 1 } + - match: { data_streams.0.generation: 2 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } - do: index: @@ -323,8 +371,107 @@ teardown: - match: { data_streams.0.name: data-stream-for-rollover } - match: { data_streams.0.timestamp_field.name: '@timestamp' } # Both backing and failure indices use the same generation field. - - match: { data_streams.0.generation: 1 } + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + +--- +"Rolling over an uninitialized failure store should initialize it": + # Initializing with conditions is not allowed. + - do: + catch: /Rolling over\/initializing an empty failure store is only supported without conditions\./ + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + body: + conditions: + max_docs: 1 + + - do: + indices.rollover: + alias: "data-stream-for-rollover" + target_failure_store: true + + - match: { acknowledged: true } + - match: { old_index: "_none_" } + - match: { new_index: "/\\.fs-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: "*" + - match: { data_streams.0.name: data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 2 } - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + +--- +"Rolling over a failure store on a data stream without the failure store enabled should work": + - do: + allowed_warnings: + - "index template [my-other-template] has index patterns [data-*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template] will take precedence during new index creation" + indices.put_index_template: + name: my-other-template + body: + index_patterns: [other-data-*] + data_stream: {} + + - do: + indices.create_data_stream: + name: other-data-stream-for-rollover + + # Initializing should work + - do: + indices.rollover: + alias: "other-data-stream-for-rollover" + target_failure_store: true + + - match: { acknowledged: true } + - match: { old_index: "_none_" } + - match: { new_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: other-data-stream-for-rollover + - match: { data_streams.0.name: other-data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 2 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 1 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + + # And "regular" rollover should work + - do: + indices.rollover: + alias: "other-data-stream-for-rollover" + target_failure_store: true + + - match: { acknowledged: true } + - match: { old_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000002/" } + - match: { new_index: "/\\.fs-other-data-stream-for-rollover-(\\d{4}\\.\\d{2}\\.\\d{2}-)?000003/" } + - match: { rolled_over: true } + - match: { dry_run: false } + + - do: + indices.get_data_stream: + name: other-data-stream-for-rollover + - match: { data_streams.0.name: other-data-stream-for-rollover } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + # Both backing and failure indices use the same generation field. + - match: { data_streams.0.generation: 3 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 2 } + - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000002/' } + - match: { data_streams.0.failure_store.indices.1.index_name: '/\.fs-other-data-stream-for-rollover-(\d{4}\.\d{2}\.\d{2}-)?000003/' } diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml index 3ab22e6271c6d..61d17c3d675cf 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/30_auto_create_data_stream.yml @@ -1,5 +1,5 @@ --- -"Put index template": +"Auto-create data stream": - requires: cluster_features: ["gte_v7.9.0"] reason: "data streams only supported in 7.9+" @@ -48,7 +48,7 @@ - is_true: acknowledged --- -"Put index template with failure store": +"Don't initialize failure store during data stream auto-creation on successful index": - requires: cluster_features: ["gte_v8.15.0"] reason: "data stream failure stores REST structure changed in 8.15+" @@ -92,8 +92,7 @@ - length: { data_streams.0.indices: 1 } - match: { data_streams.0.indices.0.index_name: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } - match: { data_streams.0.failure_store.enabled: true } - - length: { data_streams.0.failure_store.indices: 1 } - - match: { data_streams.0.failure_store.indices.0.index_name: '/\.fs-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + - length: { data_streams.0.failure_store.indices: 0 } - do: indices.delete_data_stream: diff --git a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java index 1621a235187a1..7bb875f8b6f69 100644 --- a/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java +++ b/modules/ingest-common/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverWithPipelinesIT.java @@ -12,7 +12,7 @@ import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.ingest.common.IngestCommonPlugin; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.Plugin; @@ -102,7 +102,7 @@ public DocumentSizeObserver newDocumentSizeObserver() { @Override public DocumentSizeReporter newDocumentSizeReporter( String indexName, - IndexMode indexMode, + MapperService mapperService, DocumentSizeAccumulator documentSizeAccumulator ) { return DocumentSizeReporter.EMPTY_INSTANCE; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java index 2557e8c4682ac..c7a1337f20e59 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/stats/GeoIpStatsAction.java @@ -11,7 +11,6 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.action.support.nodes.BaseNodesResponse; @@ -66,11 +65,6 @@ public boolean equals(Object obj) { } return true; } - - @Override - public void writeTo(StreamOutput out) { - TransportAction.localOnly(); - } } public static class NodeRequest extends TransportRequest { diff --git a/modules/lang-painless/src/main/generated/whitelist-json/painless-score.json b/modules/lang-painless/src/main/generated/whitelist-json/painless-score.json index da9f7f7b60386..0a067b3e98b56 100644 --- a/modules/lang-painless/src/main/generated/whitelist-json/painless-score.json +++ b/modules/lang-painless/src/main/generated/whitelist-json/painless-score.json @@ -1 +1 @@ -{"name":"score","classes":[{"name":"String","imported":true,"constructors":[{"declaring":"String","parameters":[]}],"static_methods":[{"declaring":"String","name":"copyValueOf","return":"String","parameters":["char[]"]},{"declaring":"String","name":"copyValueOf","return":"String","parameters":["char[]","int","int"]},{"declaring":"String","name":"format","return":"String","parameters":["String","def[]"]},{"declaring":"String","name":"format","return":"String","parameters":["Locale","String","def[]"]},{"declaring":"String","name":"join","return":"String","parameters":["CharSequence","Iterable"]},{"declaring":"String","name":"valueOf","return":"String","parameters":["def"]}],"methods":[{"declaring":"CharSequence","name":"charAt","return":"char","parameters":["int"]},{"declaring":"CharSequence","name":"chars","return":"IntStream","parameters":[]},{"declaring":"String","name":"codePointAt","return":"int","parameters":["int"]},{"declaring":"String","name":"codePointBefore","return":"int","parameters":["int"]},{"declaring":"String","name":"codePointCount","return":"int","parameters":["int","int"]},{"declaring":"CharSequence","name":"codePoints","return":"IntStream","parameters":[]},{"declaring":"String","name":"compareTo","return":"int","parameters":["String"]},{"declaring":"String","name":"compareToIgnoreCase","return":"int","parameters":["String"]},{"declaring":"String","name":"concat","return":"String","parameters":["String"]},{"declaring":"String","name":"contains","return":"boolean","parameters":["CharSequence"]},{"declaring":"String","name":"contentEquals","return":"boolean","parameters":["CharSequence"]},{"declaring":null,"name":"decodeBase64","return":"String","parameters":[]},{"declaring":null,"name":"encodeBase64","return":"String","parameters":[]},{"declaring":"String","name":"endsWith","return":"boolean","parameters":["String"]},{"declaring":"Object","name":"equals","return":"boolean","parameters":["Object"]},{"declaring":"String","name":"equalsIgnoreCase","return":"boolean","parameters":["String"]},{"declaring":"String","name":"getChars","return":"void","parameters":["int","int","char[]","int"]},{"declaring":"Object","name":"hashCode","return":"int","parameters":[]},{"declaring":"String","name":"indexOf","return":"int","parameters":["String"]},{"declaring":"String","name":"indexOf","return":"int","parameters":["String","int"]},{"declaring":"String","name":"isEmpty","return":"boolean","parameters":[]},{"declaring":"String","name":"lastIndexOf","return":"int","parameters":["String"]},{"declaring":"String","name":"lastIndexOf","return":"int","parameters":["String","int"]},{"declaring":"CharSequence","name":"length","return":"int","parameters":[]},{"declaring":"String","name":"offsetByCodePoints","return":"int","parameters":["int","int"]},{"declaring":"String","name":"regionMatches","return":"boolean","parameters":["int","String","int","int"]},{"declaring":"String","name":"regionMatches","return":"boolean","parameters":["boolean","int","String","int","int"]},{"declaring":"String","name":"replace","return":"String","parameters":["CharSequence","CharSequence"]},{"declaring":null,"name":"replaceAll","return":"String","parameters":["Pattern","Function"]},{"declaring":null,"name":"replaceFirst","return":"String","parameters":["Pattern","Function"]},{"declaring":null,"name":"splitOnToken","return":"String[]","parameters":["String"]},{"declaring":null,"name":"splitOnToken","return":"String[]","parameters":["String","int"]},{"declaring":"String","name":"startsWith","return":"boolean","parameters":["String"]},{"declaring":"String","name":"startsWith","return":"boolean","parameters":["String","int"]},{"declaring":"CharSequence","name":"subSequence","return":"CharSequence","parameters":["int","int"]},{"declaring":"String","name":"substring","return":"String","parameters":["int"]},{"declaring":"String","name":"substring","return":"String","parameters":["int","int"]},{"declaring":"String","name":"toCharArray","return":"char[]","parameters":[]},{"declaring":"String","name":"toLowerCase","return":"String","parameters":[]},{"declaring":"String","name":"toLowerCase","return":"String","parameters":["Locale"]},{"declaring":"CharSequence","name":"toString","return":"String","parameters":[]},{"declaring":"String","name":"toUpperCase","return":"String","parameters":[]},{"declaring":"String","name":"toUpperCase","return":"String","parameters":["Locale"]},{"declaring":"String","name":"trim","return":"String","parameters":[]}],"static_fields":[],"fields":[]},{"name":"DenseVectorScriptDocValues","imported":true,"constructors":[],"static_methods":[],"methods":[{"declaring":"Collection","name":"add","return":"boolean","parameters":["def"]},{"declaring":"List","name":"add","return":"void","parameters":["int","def"]},{"declaring":"Collection","name":"addAll","return":"boolean","parameters":["Collection"]},{"declaring":"List","name":"addAll","return":"boolean","parameters":["int","Collection"]},{"declaring":null,"name":"any","return":"boolean","parameters":["Predicate"]},{"declaring":null,"name":"asCollection","return":"Collection","parameters":[]},{"declaring":null,"name":"asList","return":"List","parameters":[]},{"declaring":"Collection","name":"clear","return":"void","parameters":[]},{"declaring":null,"name":"collect","return":"List","parameters":["Function"]},{"declaring":null,"name":"collect","return":"def","parameters":["Collection","Function"]},{"declaring":"Collection","name":"contains","return":"boolean","parameters":["def"]},{"declaring":"Collection","name":"containsAll","return":"boolean","parameters":["Collection"]},{"declaring":null,"name":"each","return":"def","parameters":["Consumer"]},{"declaring":null,"name":"eachWithIndex","return":"def","parameters":["ObjIntConsumer"]},{"declaring":"List","name":"equals","return":"boolean","parameters":["Object"]},{"declaring":null,"name":"every","return":"boolean","parameters":["Predicate"]},{"declaring":null,"name":"find","return":"def","parameters":["Predicate"]},{"declaring":null,"name":"findAll","return":"List","parameters":["Predicate"]},{"declaring":null,"name":"findResult","return":"def","parameters":["Function"]},{"declaring":null,"name":"findResult","return":"def","parameters":["def","Function"]},{"declaring":null,"name":"findResults","return":"List","parameters":["Function"]},{"declaring":"Iterable","name":"forEach","return":"void","parameters":["Consumer"]},{"declaring":"List","name":"get","return":"def","parameters":["int"]},{"declaring":null,"name":"getByPath","return":"Object","parameters":["String"]},{"declaring":null,"name":"getByPath","return":"Object","parameters":["String","Object"]},{"declaring":null,"name":"getLength","return":"int","parameters":[]},{"declaring":null,"name":"groupBy","return":"Map","parameters":["Function"]},{"declaring":"List","name":"hashCode","return":"int","parameters":[]},{"declaring":"List","name":"indexOf","return":"int","parameters":["def"]},{"declaring":"Collection","name":"isEmpty","return":"boolean","parameters":[]},{"declaring":"Iterable","name":"iterator","return":"Iterator","parameters":[]},{"declaring":null,"name":"join","return":"String","parameters":["String"]},{"declaring":"List","name":"lastIndexOf","return":"int","parameters":["def"]},{"declaring":"List","name":"listIterator","return":"ListIterator","parameters":[]},{"declaring":"List","name":"listIterator","return":"ListIterator","parameters":["int"]},{"declaring":"List","name":"remove","return":"def","parameters":["int"]},{"declaring":"Collection","name":"removeAll","return":"boolean","parameters":["Collection"]},{"declaring":"Collection","name":"removeIf","return":"boolean","parameters":["Predicate"]},{"declaring":"List","name":"replaceAll","return":"void","parameters":["UnaryOperator"]},{"declaring":"Collection","name":"retainAll","return":"boolean","parameters":["Collection"]},{"declaring":"List","name":"set","return":"def","parameters":["int","def"]},{"declaring":"Collection","name":"size","return":"int","parameters":[]},{"declaring":"List","name":"sort","return":"void","parameters":["Comparator"]},{"declaring":null,"name":"split","return":"List","parameters":["Predicate"]},{"declaring":"Collection","name":"spliterator","return":"Spliterator","parameters":[]},{"declaring":"Collection","name":"stream","return":"Stream","parameters":[]},{"declaring":"List","name":"subList","return":"List","parameters":["int","int"]},{"declaring":null,"name":"sum","return":"double","parameters":[]},{"declaring":null,"name":"sum","return":"double","parameters":["ToDoubleFunction"]},{"declaring":"Collection","name":"toArray","return":"def[]","parameters":[]},{"declaring":"Collection","name":"toArray","return":"def[]","parameters":["def[]"]},{"declaring":"Object","name":"toString","return":"String","parameters":[]}],"static_fields":[],"fields":[]},{"name":"VersionScriptDocValues","imported":true,"constructors":[],"static_methods":[],"methods":[{"declaring":"Collection","name":"add","return":"boolean","parameters":["def"]},{"declaring":"List","name":"add","return":"void","parameters":["int","def"]},{"declaring":"Collection","name":"addAll","return":"boolean","parameters":["Collection"]},{"declaring":"List","name":"addAll","return":"boolean","parameters":["int","Collection"]},{"declaring":null,"name":"any","return":"boolean","parameters":["Predicate"]},{"declaring":null,"name":"asCollection","return":"Collection","parameters":[]},{"declaring":null,"name":"asList","return":"List","parameters":[]},{"declaring":"Collection","name":"clear","return":"void","parameters":[]},{"declaring":null,"name":"collect","return":"List","parameters":["Function"]},{"declaring":null,"name":"collect","return":"def","parameters":["Collection","Function"]},{"declaring":"Collection","name":"contains","return":"boolean","parameters":["def"]},{"declaring":"Collection","name":"containsAll","return":"boolean","parameters":["Collection"]},{"declaring":null,"name":"each","return":"def","parameters":["Consumer"]},{"declaring":null,"name":"eachWithIndex","return":"def","parameters":["ObjIntConsumer"]},{"declaring":"List","name":"equals","return":"boolean","parameters":["Object"]},{"declaring":null,"name":"every","return":"boolean","parameters":["Predicate"]},{"declaring":null,"name":"find","return":"def","parameters":["Predicate"]},{"declaring":null,"name":"findAll","return":"List","parameters":["Predicate"]},{"declaring":null,"name":"findResult","return":"def","parameters":["Function"]},{"declaring":null,"name":"findResult","return":"def","parameters":["def","Function"]},{"declaring":null,"name":"findResults","return":"List","parameters":["Function"]},{"declaring":"Iterable","name":"forEach","return":"void","parameters":["Consumer"]},{"declaring":"VersionScriptDocValues","name":"get","return":"String","parameters":["int"]},{"declaring":null,"name":"getByPath","return":"Object","parameters":["String"]},{"declaring":null,"name":"getByPath","return":"Object","parameters":["String","Object"]},{"declaring":null,"name":"getLength","return":"int","parameters":[]},{"declaring":"VersionScriptDocValues","name":"getValue","return":"String","parameters":[]},{"declaring":null,"name":"groupBy","return":"Map","parameters":["Function"]},{"declaring":"List","name":"hashCode","return":"int","parameters":[]},{"declaring":"List","name":"indexOf","return":"int","parameters":["def"]},{"declaring":"Collection","name":"isEmpty","return":"boolean","parameters":[]},{"declaring":"Iterable","name":"iterator","return":"Iterator","parameters":[]},{"declaring":null,"name":"join","return":"String","parameters":["String"]},{"declaring":"List","name":"lastIndexOf","return":"int","parameters":["def"]},{"declaring":"List","name":"listIterator","return":"ListIterator","parameters":[]},{"declaring":"List","name":"listIterator","return":"ListIterator","parameters":["int"]},{"declaring":"List","name":"remove","return":"def","parameters":["int"]},{"declaring":"Collection","name":"removeAll","return":"boolean","parameters":["Collection"]},{"declaring":"Collection","name":"removeIf","return":"boolean","parameters":["Predicate"]},{"declaring":"List","name":"replaceAll","return":"void","parameters":["UnaryOperator"]},{"declaring":"Collection","name":"retainAll","return":"boolean","parameters":["Collection"]},{"declaring":"List","name":"set","return":"def","parameters":["int","def"]},{"declaring":"Collection","name":"size","return":"int","parameters":[]},{"declaring":"List","name":"sort","return":"void","parameters":["Comparator"]},{"declaring":null,"name":"split","return":"List","parameters":["Predicate"]},{"declaring":"Collection","name":"spliterator","return":"Spliterator","parameters":[]},{"declaring":"Collection","name":"stream","return":"Stream","parameters":[]},{"declaring":"List","name":"subList","return":"List","parameters":["int","int"]},{"declaring":null,"name":"sum","return":"double","parameters":[]},{"declaring":null,"name":"sum","return":"double","parameters":["ToDoubleFunction"]},{"declaring":"Collection","name":"toArray","return":"def[]","parameters":[]},{"declaring":"Collection","name":"toArray","return":"def[]","parameters":["def[]"]},{"declaring":"Object","name":"toString","return":"String","parameters":[]}],"static_fields":[],"fields":[]}],"imported_methods":[{"declaring":null,"name":"saturation","return":"double","parameters":["double","double"]},{"declaring":null,"name":"sigmoid","return":"double","parameters":["double","double","double"]}],"class_bindings":[{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity","name":"cosineSimilarity","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.lang.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayDateExp","name":"decayDateExp","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.script.JodaCompatibleZonedDateTime"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayDateGauss","name":"decayDateGauss","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.script.JodaCompatibleZonedDateTime"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayDateLinear","name":"decayDateLinear","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.script.JodaCompatibleZonedDateTime"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayGeoExp","name":"decayGeoExp","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.common.geo.GeoPoint"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayGeoGauss","name":"decayGeoGauss","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.common.geo.GeoPoint"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayGeoLinear","name":"decayGeoLinear","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.common.geo.GeoPoint"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayNumericExp","name":"decayNumericExp","return":"double","read_only":4,"parameters":["double","double","double","double","double"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayNumericGauss","name":"decayNumericGauss","return":"double","read_only":4,"parameters":["double","double","double","double","double"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayNumericLinear","name":"decayNumericLinear","return":"double","read_only":4,"parameters":["double","double","double","double","double"]},{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$DotProduct","name":"dotProduct","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.lang.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$L1Norm","name":"l1norm","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.lang.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$L2Norm","name":"l2norm","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.lang.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$RandomScoreDoc","name":"randomScore","return":"double","read_only":2,"parameters":["org.elasticsearch.script.ScoreScript","int"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$RandomScoreField","name":"randomScore","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","int","java.lang.String"]}],"instance_bindings":[]} +{"name":"score","classes":[{"name":"String","imported":true,"constructors":[{"declaring":"String","parameters":[]}],"static_methods":[{"declaring":"String","name":"copyValueOf","return":"String","parameters":["char[]"]},{"declaring":"String","name":"copyValueOf","return":"String","parameters":["char[]","int","int"]},{"declaring":"String","name":"format","return":"String","parameters":["String","def[]"]},{"declaring":"String","name":"format","return":"String","parameters":["Locale","String","def[]"]},{"declaring":"String","name":"join","return":"String","parameters":["CharSequence","Iterable"]},{"declaring":"String","name":"valueOf","return":"String","parameters":["def"]}],"methods":[{"declaring":"CharSequence","name":"charAt","return":"char","parameters":["int"]},{"declaring":"CharSequence","name":"chars","return":"IntStream","parameters":[]},{"declaring":"String","name":"codePointAt","return":"int","parameters":["int"]},{"declaring":"String","name":"codePointBefore","return":"int","parameters":["int"]},{"declaring":"String","name":"codePointCount","return":"int","parameters":["int","int"]},{"declaring":"CharSequence","name":"codePoints","return":"IntStream","parameters":[]},{"declaring":"String","name":"compareTo","return":"int","parameters":["String"]},{"declaring":"String","name":"compareToIgnoreCase","return":"int","parameters":["String"]},{"declaring":"String","name":"concat","return":"String","parameters":["String"]},{"declaring":"String","name":"contains","return":"boolean","parameters":["CharSequence"]},{"declaring":"String","name":"contentEquals","return":"boolean","parameters":["CharSequence"]},{"declaring":null,"name":"decodeBase64","return":"String","parameters":[]},{"declaring":null,"name":"encodeBase64","return":"String","parameters":[]},{"declaring":"String","name":"endsWith","return":"boolean","parameters":["String"]},{"declaring":"Object","name":"equals","return":"boolean","parameters":["Object"]},{"declaring":"String","name":"equalsIgnoreCase","return":"boolean","parameters":["String"]},{"declaring":"String","name":"getChars","return":"void","parameters":["int","int","char[]","int"]},{"declaring":"Object","name":"hashCode","return":"int","parameters":[]},{"declaring":"String","name":"indexOf","return":"int","parameters":["String"]},{"declaring":"String","name":"indexOf","return":"int","parameters":["String","int"]},{"declaring":"String","name":"isEmpty","return":"boolean","parameters":[]},{"declaring":"String","name":"lastIndexOf","return":"int","parameters":["String"]},{"declaring":"String","name":"lastIndexOf","return":"int","parameters":["String","int"]},{"declaring":"CharSequence","name":"length","return":"int","parameters":[]},{"declaring":"String","name":"offsetByCodePoints","return":"int","parameters":["int","int"]},{"declaring":"String","name":"regionMatches","return":"boolean","parameters":["int","String","int","int"]},{"declaring":"String","name":"regionMatches","return":"boolean","parameters":["boolean","int","String","int","int"]},{"declaring":"String","name":"replace","return":"String","parameters":["CharSequence","CharSequence"]},{"declaring":null,"name":"replaceAll","return":"String","parameters":["Pattern","Function"]},{"declaring":null,"name":"replaceFirst","return":"String","parameters":["Pattern","Function"]},{"declaring":null,"name":"splitOnToken","return":"String[]","parameters":["String"]},{"declaring":null,"name":"splitOnToken","return":"String[]","parameters":["String","int"]},{"declaring":"String","name":"startsWith","return":"boolean","parameters":["String"]},{"declaring":"String","name":"startsWith","return":"boolean","parameters":["String","int"]},{"declaring":"CharSequence","name":"subSequence","return":"CharSequence","parameters":["int","int"]},{"declaring":"String","name":"substring","return":"String","parameters":["int"]},{"declaring":"String","name":"substring","return":"String","parameters":["int","int"]},{"declaring":"String","name":"toCharArray","return":"char[]","parameters":[]},{"declaring":"String","name":"toLowerCase","return":"String","parameters":[]},{"declaring":"String","name":"toLowerCase","return":"String","parameters":["Locale"]},{"declaring":"CharSequence","name":"toString","return":"String","parameters":[]},{"declaring":"String","name":"toUpperCase","return":"String","parameters":[]},{"declaring":"String","name":"toUpperCase","return":"String","parameters":["Locale"]},{"declaring":"String","name":"trim","return":"String","parameters":[]}],"static_fields":[],"fields":[]},{"name":"DenseVectorScriptDocValues","imported":true,"constructors":[],"static_methods":[],"methods":[{"declaring":"Collection","name":"add","return":"boolean","parameters":["def"]},{"declaring":"List","name":"add","return":"void","parameters":["int","def"]},{"declaring":"Collection","name":"addAll","return":"boolean","parameters":["Collection"]},{"declaring":"List","name":"addAll","return":"boolean","parameters":["int","Collection"]},{"declaring":null,"name":"any","return":"boolean","parameters":["Predicate"]},{"declaring":null,"name":"asCollection","return":"Collection","parameters":[]},{"declaring":null,"name":"asList","return":"List","parameters":[]},{"declaring":"Collection","name":"clear","return":"void","parameters":[]},{"declaring":null,"name":"collect","return":"List","parameters":["Function"]},{"declaring":null,"name":"collect","return":"def","parameters":["Collection","Function"]},{"declaring":"Collection","name":"contains","return":"boolean","parameters":["def"]},{"declaring":"Collection","name":"containsAll","return":"boolean","parameters":["Collection"]},{"declaring":null,"name":"each","return":"def","parameters":["Consumer"]},{"declaring":null,"name":"eachWithIndex","return":"def","parameters":["ObjIntConsumer"]},{"declaring":"List","name":"equals","return":"boolean","parameters":["Object"]},{"declaring":null,"name":"every","return":"boolean","parameters":["Predicate"]},{"declaring":null,"name":"find","return":"def","parameters":["Predicate"]},{"declaring":null,"name":"findAll","return":"List","parameters":["Predicate"]},{"declaring":null,"name":"findResult","return":"def","parameters":["Function"]},{"declaring":null,"name":"findResult","return":"def","parameters":["def","Function"]},{"declaring":null,"name":"findResults","return":"List","parameters":["Function"]},{"declaring":"Iterable","name":"forEach","return":"void","parameters":["Consumer"]},{"declaring":"List","name":"get","return":"def","parameters":["int"]},{"declaring":null,"name":"getByPath","return":"Object","parameters":["String"]},{"declaring":null,"name":"getByPath","return":"Object","parameters":["String","Object"]},{"declaring":null,"name":"getLength","return":"int","parameters":[]},{"declaring":null,"name":"groupBy","return":"Map","parameters":["Function"]},{"declaring":"List","name":"hashCode","return":"int","parameters":[]},{"declaring":"List","name":"indexOf","return":"int","parameters":["def"]},{"declaring":"Collection","name":"isEmpty","return":"boolean","parameters":[]},{"declaring":"Iterable","name":"iterator","return":"Iterator","parameters":[]},{"declaring":null,"name":"join","return":"String","parameters":["String"]},{"declaring":"List","name":"lastIndexOf","return":"int","parameters":["def"]},{"declaring":"List","name":"listIterator","return":"ListIterator","parameters":[]},{"declaring":"List","name":"listIterator","return":"ListIterator","parameters":["int"]},{"declaring":"List","name":"remove","return":"def","parameters":["int"]},{"declaring":"Collection","name":"removeAll","return":"boolean","parameters":["Collection"]},{"declaring":"Collection","name":"removeIf","return":"boolean","parameters":["Predicate"]},{"declaring":"List","name":"replaceAll","return":"void","parameters":["UnaryOperator"]},{"declaring":"Collection","name":"retainAll","return":"boolean","parameters":["Collection"]},{"declaring":"List","name":"set","return":"def","parameters":["int","def"]},{"declaring":"Collection","name":"size","return":"int","parameters":[]},{"declaring":"List","name":"sort","return":"void","parameters":["Comparator"]},{"declaring":null,"name":"split","return":"List","parameters":["Predicate"]},{"declaring":"Collection","name":"spliterator","return":"Spliterator","parameters":[]},{"declaring":"Collection","name":"stream","return":"Stream","parameters":[]},{"declaring":"List","name":"subList","return":"List","parameters":["int","int"]},{"declaring":null,"name":"sum","return":"double","parameters":[]},{"declaring":null,"name":"sum","return":"double","parameters":["ToDoubleFunction"]},{"declaring":"Collection","name":"toArray","return":"def[]","parameters":[]},{"declaring":"Collection","name":"toArray","return":"def[]","parameters":["def[]"]},{"declaring":"Object","name":"toString","return":"String","parameters":[]}],"static_fields":[],"fields":[]},{"name":"VersionScriptDocValues","imported":true,"constructors":[],"static_methods":[],"methods":[{"declaring":"Collection","name":"add","return":"boolean","parameters":["def"]},{"declaring":"List","name":"add","return":"void","parameters":["int","def"]},{"declaring":"Collection","name":"addAll","return":"boolean","parameters":["Collection"]},{"declaring":"List","name":"addAll","return":"boolean","parameters":["int","Collection"]},{"declaring":null,"name":"any","return":"boolean","parameters":["Predicate"]},{"declaring":null,"name":"asCollection","return":"Collection","parameters":[]},{"declaring":null,"name":"asList","return":"List","parameters":[]},{"declaring":"Collection","name":"clear","return":"void","parameters":[]},{"declaring":null,"name":"collect","return":"List","parameters":["Function"]},{"declaring":null,"name":"collect","return":"def","parameters":["Collection","Function"]},{"declaring":"Collection","name":"contains","return":"boolean","parameters":["def"]},{"declaring":"Collection","name":"containsAll","return":"boolean","parameters":["Collection"]},{"declaring":null,"name":"each","return":"def","parameters":["Consumer"]},{"declaring":null,"name":"eachWithIndex","return":"def","parameters":["ObjIntConsumer"]},{"declaring":"List","name":"equals","return":"boolean","parameters":["Object"]},{"declaring":null,"name":"every","return":"boolean","parameters":["Predicate"]},{"declaring":null,"name":"find","return":"def","parameters":["Predicate"]},{"declaring":null,"name":"findAll","return":"List","parameters":["Predicate"]},{"declaring":null,"name":"findResult","return":"def","parameters":["Function"]},{"declaring":null,"name":"findResult","return":"def","parameters":["def","Function"]},{"declaring":null,"name":"findResults","return":"List","parameters":["Function"]},{"declaring":"Iterable","name":"forEach","return":"void","parameters":["Consumer"]},{"declaring":"VersionScriptDocValues","name":"get","return":"String","parameters":["int"]},{"declaring":null,"name":"getByPath","return":"Object","parameters":["String"]},{"declaring":null,"name":"getByPath","return":"Object","parameters":["String","Object"]},{"declaring":null,"name":"getLength","return":"int","parameters":[]},{"declaring":"VersionScriptDocValues","name":"getValue","return":"String","parameters":[]},{"declaring":null,"name":"groupBy","return":"Map","parameters":["Function"]},{"declaring":"List","name":"hashCode","return":"int","parameters":[]},{"declaring":"List","name":"indexOf","return":"int","parameters":["def"]},{"declaring":"Collection","name":"isEmpty","return":"boolean","parameters":[]},{"declaring":"Iterable","name":"iterator","return":"Iterator","parameters":[]},{"declaring":null,"name":"join","return":"String","parameters":["String"]},{"declaring":"List","name":"lastIndexOf","return":"int","parameters":["def"]},{"declaring":"List","name":"listIterator","return":"ListIterator","parameters":[]},{"declaring":"List","name":"listIterator","return":"ListIterator","parameters":["int"]},{"declaring":"List","name":"remove","return":"def","parameters":["int"]},{"declaring":"Collection","name":"removeAll","return":"boolean","parameters":["Collection"]},{"declaring":"Collection","name":"removeIf","return":"boolean","parameters":["Predicate"]},{"declaring":"List","name":"replaceAll","return":"void","parameters":["UnaryOperator"]},{"declaring":"Collection","name":"retainAll","return":"boolean","parameters":["Collection"]},{"declaring":"List","name":"set","return":"def","parameters":["int","def"]},{"declaring":"Collection","name":"size","return":"int","parameters":[]},{"declaring":"List","name":"sort","return":"void","parameters":["Comparator"]},{"declaring":null,"name":"split","return":"List","parameters":["Predicate"]},{"declaring":"Collection","name":"spliterator","return":"Spliterator","parameters":[]},{"declaring":"Collection","name":"stream","return":"Stream","parameters":[]},{"declaring":"List","name":"subList","return":"List","parameters":["int","int"]},{"declaring":null,"name":"sum","return":"double","parameters":[]},{"declaring":null,"name":"sum","return":"double","parameters":["ToDoubleFunction"]},{"declaring":"Collection","name":"toArray","return":"def[]","parameters":[]},{"declaring":"Collection","name":"toArray","return":"def[]","parameters":["def[]"]},{"declaring":"Object","name":"toString","return":"String","parameters":[]}],"static_fields":[],"fields":[]}],"imported_methods":[{"declaring":null,"name":"saturation","return":"double","parameters":["double","double"]},{"declaring":null,"name":"sigmoid","return":"double","parameters":["double","double","double"]}],"class_bindings":[{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity","name":"cosineSimilarity","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.lang.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayDateExp","name":"decayDateExp","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.script.JodaCompatibleZonedDateTime"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayDateGauss","name":"decayDateGauss","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.script.JodaCompatibleZonedDateTime"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayDateLinear","name":"decayDateLinear","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.script.JodaCompatibleZonedDateTime"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayGeoExp","name":"decayGeoExp","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.common.geo.GeoPoint"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayGeoGauss","name":"decayGeoGauss","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.common.geo.GeoPoint"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayGeoLinear","name":"decayGeoLinear","return":"double","read_only":4,"parameters":["java.lang.String","java.lang.String","java.lang.String","double","org.elasticsearch.common.geo.GeoPoint"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayNumericExp","name":"decayNumericExp","return":"double","read_only":4,"parameters":["double","double","double","double","double"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayNumericGauss","name":"decayNumericGauss","return":"double","read_only":4,"parameters":["double","double","double","double","double"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$DecayNumericLinear","name":"decayNumericLinear","return":"double","read_only":4,"parameters":["double","double","double","double","double"]},{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$DotProduct","name":"dotProduct","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.lang.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$L1Norm","name":"l1norm","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.lang.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$Hamming","name":"hamming","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.util.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.VectorScoreScriptUtils$L2Norm","name":"l2norm","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","java.lang.Object","java.lang.String"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$RandomScoreDoc","name":"randomScore","return":"double","read_only":2,"parameters":["org.elasticsearch.script.ScoreScript","int"]},{"declaring":"org.elasticsearch.script.ScoreScriptUtils$RandomScoreField","name":"randomScore","return":"double","read_only":3,"parameters":["org.elasticsearch.script.ScoreScript","int","java.lang.String"]}],"instance_bindings":[]} diff --git a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt index b0506e7aa677a..5082d5f1c7bdb 100644 --- a/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt +++ b/modules/lang-painless/src/main/resources/org/elasticsearch/painless/org.elasticsearch.script.score.txt @@ -31,5 +31,6 @@ static_import { double l2norm(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$L2Norm double cosineSimilarity(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$CosineSimilarity double dotProduct(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$DotProduct + double hamming(org.elasticsearch.script.ScoreScript, Object, String) bound_to org.elasticsearch.script.VectorScoreScriptUtils$Hamming } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml index a4245621f83e0..e49dc20e73406 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/140_dense_vector_basic.yml @@ -219,3 +219,36 @@ setup: - match: {hits.hits.2._id: "2"} - close_to: {hits.hits.2._score: {value: 186.34454, error: 0.01}} +--- +"Test hamming distance fails on float": + - requires: + cluster_features: ["script.hamming"] + reason: "support for hamming distance added in 8.15" + - do: + headers: + Content-Type: application/json + catch: bad_request + search: + body: + query: + script_score: + query: {match_all: {} } + script: + source: "hamming(params.query_vector, 'vector')" + params: + query_vector: [0.5, 111.3, -13.0, 14.8, -156.0] + + - do: + headers: + Content-Type: application/json + catch: bad_request + search: + body: + query: + script_score: + query: {match_all: {} } + script: + source: "hamming(params.query_vector, 'indexed_vector')" + params: + query_vector: [0.5, 111.3, -13.0, 14.8, -156.0] + diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml new file mode 100644 index 0000000000000..373f048e7be78 --- /dev/null +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/151_dense_vector_byte_hamming.yml @@ -0,0 +1,156 @@ +setup: + - requires: + cluster_features: ["script.hamming"] + reason: "support for hamming distance added in 8.15" + test_runner_features: headers + + - do: + indices.create: + index: test-index + body: + settings: + number_of_replicas: 0 + mappings: + properties: + my_dense_vector: + index: false + type: dense_vector + element_type: byte + dims: 5 + my_dense_vector_indexed: + index: true + type: dense_vector + element_type: byte + dims: 5 + + - do: + index: + index: test-index + id: "1" + body: + my_dense_vector: [8, 5, -15, 1, -7] + my_dense_vector_indexed: [8, 5, -15, 1, -7] + + - do: + index: + index: test-index + id: "2" + body: + my_dense_vector: [-1, 115, -3, 4, -128] + my_dense_vector_indexed: [-1, 115, -3, 4, -128] + + - do: + index: + index: test-index + id: "3" + body: + my_dense_vector: [2, 18, -5, 0, -124] + my_dense_vector_indexed: [2, 18, -5, 0, -124] + + - do: + indices.refresh: {} + +--- +"Hamming distance": + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "hamming(params.query_vector, 'my_dense_vector')" + params: + query_vector: [0, 111, -13, 14, -124] + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0._score: 17.0} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1._score: 16.0} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2._score: 11.0} + + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "hamming(params.query_vector, 'my_dense_vector_indexed')" + params: + query_vector: [0, 111, -13, 14, -124] + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0._score: 17.0} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1._score: 16.0} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2._score: 11.0} +--- +"Hamming distance hexidecimal": + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "hamming(params.query_vector, 'my_dense_vector')" + params: + query_vector: "006ff30e84" + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0._score: 17.0} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1._score: 16.0} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2._score: 11.0} + + + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "hamming(params.query_vector, 'my_dense_vector_indexed')" + params: + query_vector: "006ff30e84" + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0._score: 17.0} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1._score: 16.0} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2._score: 11.0} diff --git a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java index 4ef2b2e07bb26..4678215dd5b60 100644 --- a/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java +++ b/modules/legacy-geo/src/main/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapper.java @@ -58,7 +58,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -379,18 +378,18 @@ private LegacyGeoShapeParser() {} public void parse( XContentParser parser, CheckedConsumer, IOException> consumer, - Consumer onMalformed + MalformedValueHandler malformedHandler ) throws IOException { try { if (parser.currentToken() == XContentParser.Token.START_ARRAY) { while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - parse(parser, consumer, onMalformed); + parse(parser, consumer, malformedHandler); } } else { consumer.accept(ShapeParser.parse(parser)); } } catch (ElasticsearchParseException e) { - onMalformed.accept(e); + malformedHandler.notify(e); } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 14c180dc0b65c..f472ce0855625 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; @@ -196,7 +197,14 @@ public ScaledFloatFieldMapper build(MapperBuilderContext context) { metric.getValue(), indexMode ); - return new ScaledFloatFieldMapper(name(), type, multiFieldsBuilder.build(this, context), copyTo, this); + return new ScaledFloatFieldMapper( + name(), + type, + multiFieldsBuilder.build(this, context), + copyTo, + context.isSourceSynthetic(), + this + ); } } @@ -452,6 +460,7 @@ public String toString() { private final boolean stored; private final Double nullValue; private final double scalingFactor; + private final boolean isSourceSynthetic; private final boolean ignoreMalformedByDefault; private final boolean coerceByDefault; @@ -463,9 +472,11 @@ private ScaledFloatFieldMapper( ScaledFloatFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, + boolean isSourceSynthetic, Builder builder ) { super(simpleName, mappedFieldType, multiFields, copyTo); + this.isSourceSynthetic = isSourceSynthetic; this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); this.stored = builder.stored.getValue(); @@ -518,6 +529,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } catch (IllegalArgumentException e) { if (ignoreMalformed.value()) { context.addIgnoredField(mappedFieldType.name()); + if (isSourceSynthetic) { + // Save a copy of the field so synthetic source can load it + context.doc().add(IgnoreMalformedStoredValues.storedField(name(), context.parser())); + } return; } else { throw e; @@ -542,6 +557,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (Double.isFinite(doubleValue) == false) { if (ignoreMalformed.value()) { context.addIgnoredField(mappedFieldType.name()); + if (isSourceSynthetic) { + // Save a copy of the field so synthetic source can load it + context.doc().add(IgnoreMalformedStoredValues.storedField(name(), context.parser())); + } return; } else { // since we encode to a long, we have no way to carry NaNs and infinities @@ -705,11 +724,6 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values" ); } - if (ignoreMalformed.value()) { - throw new IllegalArgumentException( - "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers" - ); - } if (copyTo.copyToFields().isEmpty() != true) { throw new IllegalArgumentException( "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java index 253df4de999db..56b9bb7f748b4 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java @@ -13,7 +13,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DocumentMapper; @@ -38,7 +37,10 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; import static java.util.Collections.singletonList; import static org.hamcrest.Matchers.containsString; @@ -239,7 +241,8 @@ protected List exampleMalformedValues() { exampleMalformedValue("a").errorMatches("For input string: \"a\""), exampleMalformedValue("NaN").errorMatches("[scaled_float] only supports finite values, but got [NaN]"), exampleMalformedValue("Infinity").errorMatches("[scaled_float] only supports finite values, but got [Infinity]"), - exampleMalformedValue("-Infinity").errorMatches("[scaled_float] only supports finite values, but got [-Infinity]") + exampleMalformedValue("-Infinity").errorMatches("[scaled_float] only supports finite values, but got [-Infinity]"), + exampleMalformedValue(b -> b.value(true)).errorMatches("Current token (VALUE_TRUE) not numeric") ); } @@ -361,35 +364,62 @@ protected Object generateRandomInputValue(MappedFieldType ft) { @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { - assumeFalse("scaled_float doesn't support ignore_malformed with synthetic _source", ignoreMalformed); - return new ScaledFloatSyntheticSourceSupport(); + return new ScaledFloatSyntheticSourceSupport(ignoreMalformed); } private static class ScaledFloatSyntheticSourceSupport implements SyntheticSourceSupport { + private final boolean ignoreMalformedEnabled; private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false); private final Double nullValue = usually() ? null : round(randomValue()); + private ScaledFloatSyntheticSourceSupport(boolean ignoreMalformedEnabled) { + this.ignoreMalformedEnabled = ignoreMalformedEnabled; + } + @Override public SyntheticSourceExample example(int maxValues) { if (randomBoolean()) { - Tuple v = generateValue(); - return new SyntheticSourceExample(v.v1(), v.v2(), roundDocValues(v.v2()), this::mapping); + Value v = generateValue(); + if (v.malformedOutput == null) { + return new SyntheticSourceExample(v.input, v.output, roundDocValues(v.output), this::mapping); + } + return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping); } - List> values = randomList(1, maxValues, this::generateValue); - List in = values.stream().map(Tuple::v1).toList(); - List outList = values.stream().map(Tuple::v2).sorted().toList(); + List values = randomList(1, maxValues, this::generateValue); + List in = values.stream().map(Value::input).toList(); + + List outputFromDocValues = values.stream().filter(v -> v.malformedOutput == null).map(Value::output).sorted().toList(); + Stream malformedOutput = values.stream().filter(v -> v.malformedOutput != null).map(Value::malformedOutput); + + // Malformed values are always last in the implementation. + List outList = Stream.concat(outputFromDocValues.stream(), malformedOutput).toList(); Object out = outList.size() == 1 ? outList.get(0) : outList; - List outBlockList = values.stream().map(v -> roundDocValues(v.v2())).sorted().toList(); + + List outBlockList = outputFromDocValues.stream().map(this::roundDocValues).sorted().toList(); Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; return new SyntheticSourceExample(in, out, outBlock, this::mapping); } - private Tuple generateValue() { + private record Value(Object input, Double output, Object malformedOutput) {} + + private Value generateValue() { if (nullValue != null && randomBoolean()) { - return Tuple.tuple(null, nullValue); + return new Value(null, nullValue, null); + } + // Examples in #exampleMalformedValues() are also tested with synthetic source + // so this is not an exhaustive list. + // Here we mostly want to verify behavior of arrays that contain malformed + // values since there are modifications specific to synthetic source. + if (ignoreMalformedEnabled && randomBoolean()) { + List> choices = List.of( + () -> randomAlphaOfLengthBetween(1, 10), + () -> Map.of(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10)) + ); + var malformedInput = randomFrom(choices).get(); + return new Value(malformedInput, null, malformedInput); } double d = randomValue(); - return Tuple.tuple(d, round(d)); + return new Value(d, round(d), null); } private double round(double d) { @@ -433,6 +463,9 @@ private void mapping(XContentBuilder b) throws IOException { if (rarely()) { b.field("store", false); } + if (ignoreMalformedEnabled) { + b.field("ignore_malformed", true); + } } @Override @@ -441,10 +474,6 @@ public List invalidExample() throws IOException { new SyntheticSourceInvalidExample( equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"), b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false) - ), - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it ignores malformed numbers"), - b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("ignore_malformed", true) ) ); } diff --git a/modules/repository-azure/build.gradle b/modules/repository-azure/build.gradle index c2568d9a4db2c..d093816acd45f 100644 --- a/modules/repository-azure/build.gradle +++ b/modules/repository-azure/build.gradle @@ -21,7 +21,7 @@ versions << [ 'azureCommon': '12.19.1', 'azureCore': '1.34.0', 'azureCoreHttpNetty': '1.12.7', - 'azureJackson': '2.13.4', + 'azureJackson': '2.15.4', 'azureJacksonDatabind': '2.13.4.2', 'azureAvro': '12.5.3', diff --git a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java index f8503bca3ec67..640293ecb80b0 100644 --- a/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java +++ b/modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryMetricsTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.collect.Iterators; @@ -23,6 +22,7 @@ import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; +import org.elasticsearch.repositories.blobstore.RequestedRangeNotSatisfiedException; import org.elasticsearch.repositories.s3.S3BlobStore.Operation; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.telemetry.Measurement; @@ -39,6 +39,7 @@ import static org.elasticsearch.repositories.RepositoriesMetrics.HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_HISTOGRAM; +import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_EXCEPTIONS_TOTAL; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_OPERATIONS_TOTAL; import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_REQUESTS_TOTAL; @@ -47,8 +48,10 @@ import static org.elasticsearch.repositories.RepositoriesMetrics.METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL; import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.elasticsearch.rest.RestStatus.NOT_FOUND; +import static org.elasticsearch.rest.RestStatus.REQUESTED_RANGE_NOT_SATISFIED; import static org.elasticsearch.rest.RestStatus.TOO_MANY_REQUESTS; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; @SuppressForbidden(reason = "this test uses a HttpServer to emulate an S3 endpoint") // Need to set up a new cluster for each test because cluster settings use randomized authentication settings @@ -80,22 +83,29 @@ protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { .build(); } - public void testMetricsWithErrors() throws IOException { - final String repository = createRepository(randomRepositoryName()); - - final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); - final var blobStoreRepository = (BlobStoreRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName) - .repository(repository); - final BlobStore blobStore = blobStoreRepository.blobStore(); - final TestTelemetryPlugin plugin = internalCluster().getInstance(PluginsService.class, dataNodeName) + private static TestTelemetryPlugin getPlugin(String dataNodeName) { + var plugin = internalCluster().getInstance(PluginsService.class, dataNodeName) .filterPlugins(TestTelemetryPlugin.class) .findFirst() .orElseThrow(); - plugin.resetMeter(); + return plugin; + } + + private static BlobContainer getBlobContainer(String dataNodeName, String repository) { + final var blobStoreRepository = (BlobStoreRepository) internalCluster().getInstance(RepositoriesService.class, dataNodeName) + .repository(repository); + return blobStoreRepository.blobStore().blobContainer(BlobPath.EMPTY.add(randomIdentifier())); + } + + public void testMetricsWithErrors() throws IOException { + final String repository = createRepository(randomRepositoryName()); + + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final TestTelemetryPlugin plugin = getPlugin(dataNodeName); final OperationPurpose purpose = randomFrom(OperationPurpose.values()); - final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.EMPTY.add(randomIdentifier())); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); final String blobName = randomIdentifier(); // Put a blob @@ -132,6 +142,9 @@ public void testMetricsWithErrors() throws IOException { assertThat(getLongHistogramValue(plugin, METRIC_EXCEPTIONS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); + + // Make sure we don't hit the request range not satisfied counters + assertThat(getLongCounterValue(plugin, METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, Operation.GET_OBJECT), equalTo(0L)); } // List retry exhausted @@ -166,6 +179,39 @@ public void testMetricsWithErrors() throws IOException { assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.DELETE_OBJECTS), equalTo(1L)); } + public void testMetricsForRequestRangeNotSatisfied() { + final String repository = createRepository(randomRepositoryName()); + final String dataNodeName = internalCluster().getNodeNameThat(DiscoveryNode::canContainData); + final BlobContainer blobContainer = getBlobContainer(dataNodeName, repository); + final TestTelemetryPlugin plugin = getPlugin(dataNodeName); + + final OperationPurpose purpose = randomFrom(OperationPurpose.values()); + final String blobName = randomIdentifier(); + + for (int i = 0; i < randomIntBetween(1, 3); i++) { + final long batch = i + 1; + addErrorStatus(TOO_MANY_REQUESTS, TOO_MANY_REQUESTS, REQUESTED_RANGE_NOT_SATISFIED); + try { + blobContainer.readBlob(purpose, blobName).close(); + } catch (Exception e) { + assertThat(e, instanceOf(RequestedRangeNotSatisfiedException.class)); + } + + assertThat(getLongCounterValue(plugin, METRIC_REQUESTS_TOTAL, Operation.GET_OBJECT), equalTo(3 * batch)); + assertThat(getLongCounterValue(plugin, METRIC_OPERATIONS_TOTAL, Operation.GET_OBJECT), equalTo(batch)); + assertThat(getLongCounterValue(plugin, METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL, Operation.GET_OBJECT), equalTo(batch)); + assertThat(getLongCounterValue(plugin, METRIC_EXCEPTIONS_TOTAL, Operation.GET_OBJECT), equalTo(batch)); + assertThat(getLongHistogramValue(plugin, METRIC_EXCEPTIONS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); + assertThat( + getLongCounterValue(plugin, METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, Operation.GET_OBJECT), + equalTo(batch) + ); + assertThat(getLongCounterValue(plugin, METRIC_THROTTLES_TOTAL, Operation.GET_OBJECT), equalTo(2 * batch)); + assertThat(getLongHistogramValue(plugin, METRIC_THROTTLES_HISTOGRAM, Operation.GET_OBJECT), equalTo(2 * batch)); + assertThat(getNumberOfMeasurements(plugin, HTTP_REQUEST_TIME_IN_MICROS_HISTOGRAM, Operation.GET_OBJECT), equalTo(batch)); + } + } + private void addErrorStatus(RestStatus... statuses) { errorStatusQueue.addAll(Arrays.asList(statuses)); } diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index 2aff610dc82e9..5af53364fb765 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -52,6 +52,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.rest.RestStatus.REQUESTED_RANGE_NOT_SATISFIED; class S3BlobStore implements BlobStore { @@ -177,6 +178,23 @@ public final void collectMetrics(Request request, Response response) { .map(List::size) .orElse(0); + if (exceptionCount > 0) { + final List statusCodes = Objects.requireNonNullElse( + awsRequestMetrics.getProperty(AWSRequestMetrics.Field.StatusCode), + List.of() + ); + // REQUESTED_RANGE_NOT_SATISFIED errors are expected errors due to RCO + // TODO Add more expected client error codes? + final long amountOfRequestRangeNotSatisfiedErrors = statusCodes.stream() + .filter(e -> (Integer) e == REQUESTED_RANGE_NOT_SATISFIED.getStatus()) + .count(); + if (amountOfRequestRangeNotSatisfiedErrors > 0) { + s3RepositoriesMetrics.common() + .requestRangeNotSatisfiedExceptionCounter() + .incrementBy(amountOfRequestRangeNotSatisfiedErrors, attributes); + } + } + s3RepositoriesMetrics.common().operationCounter().incrementBy(1, attributes); if (numberOfAwsErrors == requestCount) { s3RepositoriesMetrics.common().unsuccessfulOperationCounter().incrementBy(1, attributes); diff --git a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java index 77333677120a9..5ad1152d65e85 100644 --- a/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java +++ b/modules/transport-netty4/src/internalClusterTest/java/org/elasticsearch/http/netty4/Netty4ChunkedContinuationsIT.java @@ -310,7 +310,7 @@ public void onFailure(Exception exception) { safeSleep(scaledRandomIntBetween(10, 500)); // make it more likely the request started executing } cancellable.cancel(); - } // closing the request tracker ensures that everything is released, including all response chunks and the overall response + } // closing the resource tracker ensures that everything is released, including all response chunks and the overall response } private static Releasable withResourceTracker() { @@ -525,6 +525,7 @@ public ActionRequestValidationException validate() { public static class Response extends ActionResponse { private final Executor executor; volatile boolean computingContinuation; + boolean recursive = false; public Response(Executor executor) { this.executor = executor; @@ -551,11 +552,17 @@ public boolean isLastPart() { @Override public void getNextPart(ActionListener listener) { - computingContinuation = true; - executor.execute(ActionRunnable.supply(listener, () -> { - computingContinuation = false; - return getResponseBodyPart(); - })); + assertFalse(recursive); + recursive = true; + try { + computingContinuation = true; + executor.execute(ActionRunnable.supply(listener, () -> { + computingContinuation = false; + return getResponseBodyPart(); + })); + } finally { + recursive = false; + } } @Override @@ -585,7 +592,10 @@ public TransportInfiniteContinuationsAction(ActionFilters actionFilters, Transpo @Override protected void doExecute(Task task, Request request, ActionListener listener) { executor.execute( - ActionRunnable.supply(ActionTestUtils.assertNoFailureListener(listener::onResponse), () -> new Response(executor)) + ActionRunnable.supply( + ActionTestUtils.assertNoFailureListener(listener::onResponse), + () -> new Response(randomFrom(executor, EsExecutors.DIRECT_EXECUTOR_SERVICE)) + ) ); } } diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java index cfbd9ad68a317..c9beeef246703 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java @@ -29,6 +29,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ContextPreservingActionListener; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.ReleasableBytesReference; import org.elasticsearch.common.network.ThreadWatchdog; @@ -271,45 +272,59 @@ private void finishChunkedWrite() { writeSequence++; finishingWrite.combiner().finish(finishingWrite.onDone()); } else { + final var threadContext = serverTransport.getThreadPool().getThreadContext(); + assert Transports.assertDefaultThreadContext(threadContext); final var channel = finishingWrite.onDone().channel(); - ActionListener.run(ActionListener.assertOnce(new ActionListener<>() { - @Override - public void onResponse(ChunkedRestResponseBodyPart continuation) { - channel.writeAndFlush( - new Netty4ChunkedHttpContinuation(writeSequence, continuation, finishingWrite.combiner()), - finishingWrite.onDone() // pass the terminal listener/promise along the line - ); - checkShutdown(); - } - - @Override - public void onFailure(Exception e) { - logger.error( - Strings.format("failed to get continuation of HTTP response body for [%s], closing connection", channel), - e - ); - channel.close().addListener(ignored -> { - finishingWrite.combiner().add(channel.newFailedFuture(e)); - finishingWrite.combiner().finish(finishingWrite.onDone()); - }); - checkShutdown(); - } - - private void checkShutdown() { - if (channel.eventLoop().isShuttingDown()) { - // The event loop is shutting down, and https://github.com/netty/netty/issues/8007 means that we cannot know if the - // preceding activity made it onto its queue before shutdown or whether it will just vanish without a trace, so - // to avoid a leak we must double-check that the final listener is completed once the event loop is terminated. - // Note that the final listener came from Netty4Utils#safeWriteAndFlush so its executor is an ImmediateEventExecutor - // which means this completion is not subject to the same issue, it still works even if the event loop has already - // terminated. - channel.eventLoop() - .terminationFuture() - .addListener(ignored -> finishingWrite.onDone().tryFailure(new ClosedChannelException())); - } - } - - }), finishingWriteBodyPart::getNextPart); + ActionListener.run( + new ContextPreservingActionListener<>( + threadContext.newRestorableContext(false), + ActionListener.assertOnce(new ActionListener<>() { + @Override + public void onResponse(ChunkedRestResponseBodyPart continuation) { + // always fork a fresh task to avoid stack overflow + assert Transports.assertDefaultThreadContext(threadContext); + channel.eventLoop() + .execute( + () -> channel.writeAndFlush( + new Netty4ChunkedHttpContinuation(writeSequence, continuation, finishingWrite.combiner()), + finishingWrite.onDone() // pass the terminal listener/promise along the line + ) + ); + checkShutdown(); + } + + @Override + public void onFailure(Exception e) { + assert Transports.assertDefaultThreadContext(threadContext); + logger.error( + Strings.format("failed to get continuation of HTTP response body for [%s], closing connection", channel), + e + ); + channel.close().addListener(ignored -> { + finishingWrite.combiner().add(channel.newFailedFuture(e)); + finishingWrite.combiner().finish(finishingWrite.onDone()); + }); + checkShutdown(); + } + + private void checkShutdown() { + if (channel.eventLoop().isShuttingDown()) { + // The event loop is shutting down, and https://github.com/netty/netty/issues/8007 means that we cannot know + // if the preceding activity made it onto its queue before shutdown or whether it will just vanish without a + // trace, so to avoid a leak we must double-check that the final listener is completed once the event loop + // is terminated. Note that the final listener came from Netty4Utils#safeWriteAndFlush so its executor is an + // ImmediateEventExecutor which means this completion is not subject to the same issue, it still works even + // if the event loop has already terminated. + channel.eventLoop() + .terminationFuture() + .addListener(ignored -> finishingWrite.onDone().tryFailure(new ClosedChannelException())); + } + } + + }) + ), + finishingWriteBodyPart::getNextPart + ); } } diff --git a/muted-tests.yml b/muted-tests.yml index d46f710420f48..8e9ed04038074 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -5,12 +5,6 @@ tests: - class: "org.elasticsearch.cluster.coordination.CoordinatorVotingConfigurationTests" issue: "https://github.com/elastic/elasticsearch/issues/108729" method: "testClusterUUIDLogging" -- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT" - issue: "https://github.com/elastic/elasticsearch/issues/108808" - method: "test {k8s-metrics.MetricsWithAggs}" -- class: "org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT" - issue: "https://github.com/elastic/elasticsearch/issues/108809" - method: "test {k8s-metrics.MetricsWithoutAggs}" - class: "org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinderTests" issue: "https://github.com/elastic/elasticsearch/issues/108855" method: "testGuessIsDayFirstFromLocale" @@ -56,12 +50,32 @@ tests: - class: "org.elasticsearch.xpack.inference.InferenceCrudIT" issue: "https://github.com/elastic/elasticsearch/issues/109391" method: "testDeleteEndpointWhileReferencedByPipeline" -- class: org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppendTests - method: testEvaluateBlockWithoutNulls {TestCase=, } - issue: https://github.com/elastic/elasticsearch/issues/109409 - class: "org.elasticsearch.xpack.rollup.job.RollupIndexerStateTests" issue: "https://github.com/elastic/elasticsearch/issues/109627" method: "testMultipleJobTriggering" +- class: "org.elasticsearch.index.store.FsDirectoryFactoryTests" + issue: "https://github.com/elastic/elasticsearch/issues/109681" +- class: "org.elasticsearch.xpack.test.rest.XPackRestIT" + issue: "https://github.com/elastic/elasticsearch/issues/109687" + method: "test {p0=sql/translate/Translate SQL}" +- class: "org.elasticsearch.xpack.esql.EsqlAsyncSecurityIT" + issue: "https://github.com/elastic/elasticsearch/issues/109806" + method: "testInsufficientPrivilege" +- class: org.elasticsearch.action.search.SearchProgressActionListenerIT + method: testSearchProgressWithHits + issue: https://github.com/elastic/elasticsearch/issues/109830 +- class: "org.elasticsearch.xpack.shutdown.NodeShutdownReadinessIT" + issue: "https://github.com/elastic/elasticsearch/issues/109838" + method: "testShutdownReadinessService" +- class: "org.elasticsearch.xpack.security.ScrollHelperIntegTests" + issue: "https://github.com/elastic/elasticsearch/issues/109905" + method: "testFetchAllEntities" +- class: "org.elasticsearch.xpack.ml.integration.AutodetectMemoryLimitIT" + issue: "https://github.com/elastic/elasticsearch/issues/109904" +- class: "org.elasticsearch.xpack.esql.action.AsyncEsqlQueryActionIT" + issue: "https://github.com/elastic/elasticsearch/issues/109944" + method: "testBasicAsyncExecution" + # Examples: # @@ -86,3 +100,10 @@ tests: # - class: org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIPTests # method: testCrankyEvaluateBlockWithoutNulls # issue: https://github.com/elastic/elasticsearch/... +# +# Mute a single test in an ES|QL csv-spec test file: +# - class: "org.elasticsearch.xpack.esql.CsvTests" +# method: "test {union_types.MultiIndexIpStringStatsInline}" +# issue: "https://github.com/elastic/elasticsearch/..." +# Note that this mutes for the unit-test-like CsvTests only. +# Muting for the integration tests needs to be done for each IT class individually. diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java index 5c38fa36a6640..bff48ba74e8fc 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/PackageTests.java @@ -194,7 +194,7 @@ public void test50Remove() throws Exception { } assertThat(sh.runIgnoreExitCode("systemctl status elasticsearch.service").exitCode(), is(statusExitCode)); - assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode(), is(1)); + assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode(), not(0)); } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java index 19a9d9b74048e..18537c744f8b2 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/test/RpmPreservationTests.java @@ -28,7 +28,7 @@ import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation; import static org.elasticsearch.packaging.util.Platforms.isSystemd; import static org.elasticsearch.packaging.util.ServerUtils.enableGeoIpDownloader; -import static org.hamcrest.core.Is.is; +import static org.hamcrest.Matchers.not; import static org.junit.Assume.assumeTrue; public class RpmPreservationTests extends PackagingTestCase { @@ -78,7 +78,7 @@ public void test30PreserveConfig() throws Exception { assertRemoved(distribution()); if (isSystemd()) { - assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode(), is(1)); + assertThat(sh.runIgnoreExitCode("systemctl is-enabled elasticsearch.service").exitCode(), not(0)); } assertPathsDoNotExist( diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java index 21dbad9487d4e..544cd652741c8 100644 --- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java +++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/VectorSearchIT.java @@ -32,9 +32,11 @@ public VectorSearchIT(@Name("upgradedNodes") int upgradedNodes) { private static final String SCRIPT_BYTE_INDEX_NAME = "script_byte_vector_index"; private static final String BYTE_INDEX_NAME = "byte_vector_index"; private static final String QUANTIZED_INDEX_NAME = "quantized_vector_index"; + private static final String FLAT_QUANTIZED_INDEX_NAME = "flat_quantized_vector_index"; private static final String FLOAT_VECTOR_SEARCH_VERSION = "8.4.0"; private static final String BYTE_VECTOR_SEARCH_VERSION = "8.6.0"; private static final String QUANTIZED_VECTOR_SEARCH_VERSION = "8.12.1"; + private static final String FLAT_QUANTIZED_VECTOR_SEARCH_VERSION = "8.13.0"; public void testScriptByteVectorSearch() throws Exception { assumeTrue("byte vector search is not supported on this version", getOldClusterTestVersion().onOrAfter(BYTE_VECTOR_SEARCH_VERSION)); @@ -54,8 +56,6 @@ public void testScriptByteVectorSearch() throws Exception { """; createIndex(SCRIPT_BYTE_INDEX_NAME, Settings.EMPTY, mapping); indexVectors(SCRIPT_BYTE_INDEX_NAME); - // refresh the index - client().performRequest(new Request("POST", "/" + SCRIPT_BYTE_INDEX_NAME + "/_refresh")); } // search with a script query Request searchRequest = new Request("POST", "/" + SCRIPT_BYTE_INDEX_NAME + "/_search"); @@ -105,8 +105,6 @@ public void testScriptVectorSearch() throws Exception { """; createIndex(SCRIPT_VECTOR_INDEX_NAME, Settings.EMPTY, mapping); indexVectors(SCRIPT_VECTOR_INDEX_NAME); - // refresh the index - client().performRequest(new Request("POST", "/" + SCRIPT_VECTOR_INDEX_NAME + "/_refresh")); } // search with a script query Request searchRequest = new Request("POST", "/" + SCRIPT_VECTOR_INDEX_NAME + "/_search"); @@ -235,7 +233,6 @@ public void testByteVectorSearch() throws Exception { // create index and index 10 random floating point vectors createIndex(BYTE_INDEX_NAME, Settings.EMPTY, mapping); indexVectors(BYTE_INDEX_NAME); - // refresh the index // force merge the index client().performRequest(new Request("POST", "/" + BYTE_INDEX_NAME + "/_forcemerge?max_num_segments=1")); } @@ -359,6 +356,78 @@ public void testQuantizedVectorSearch() throws Exception { assertThat((double) hits.get(0).get("_score"), closeTo(0.9934857, 0.005)); } + public void testFlatQuantizedVectorSearch() throws Exception { + assumeTrue( + "Quantized vector search is not supported on this version", + getOldClusterTestVersion().onOrAfter(FLAT_QUANTIZED_VECTOR_SEARCH_VERSION) + ); + if (isOldCluster()) { + String mapping = """ + { + "properties": { + "vector": { + "type": "dense_vector", + "dims": 3, + "index": true, + "similarity": "cosine", + "index_options": { + "type": "int8_flat" + } + } + } + } + """; + // create index and index 10 random floating point vectors + createIndex(FLAT_QUANTIZED_INDEX_NAME, Settings.EMPTY, mapping); + indexVectors(FLAT_QUANTIZED_INDEX_NAME); + // force merge the index + client().performRequest(new Request("POST", "/" + FLAT_QUANTIZED_INDEX_NAME + "/_forcemerge?max_num_segments=1")); + } + Request searchRequest = new Request("POST", "/" + FLAT_QUANTIZED_INDEX_NAME + "/_search"); + searchRequest.setJsonEntity(""" + { + "query": { + "script_score": { + "query": { + "exists": { + "field": "vector" + } + }, + "script": { + "source": "cosineSimilarity(params.query, 'vector') + 1.0", + "params": { + "query": [4, 5, 6] + } + } + } + } + } + """); + Map response = search(searchRequest); + assertThat(extractValue(response, "hits.total.value"), equalTo(7)); + List> hits = extractValue(response, "hits.hits"); + assertThat(hits.get(0).get("_id"), equalTo("0")); + assertThat((double) hits.get(0).get("_score"), closeTo(1.9869276, 0.0001)); + + // search with knn + searchRequest = new Request("POST", "/" + FLAT_QUANTIZED_INDEX_NAME + "/_search"); + searchRequest.setJsonEntity(""" + { + "knn": { + "field": "vector", + "query_vector": [4, 5, 6], + "k": 2, + "num_candidates": 5 + } + } + """); + response = search(searchRequest); + assertThat(extractValue(response, "hits.total.value"), equalTo(2)); + hits = extractValue(response, "hits.hits"); + assertThat(hits.get(0).get("_id"), equalTo("0")); + assertThat((double) hits.get(0).get("_score"), closeTo(0.9934857, 0.005)); + } + private void indexVectors(String indexName) throws Exception { String[] vectors = new String[] { "{\"vector\":[1, 1, 1]}", @@ -374,6 +443,8 @@ private void indexVectors(String indexName) throws Exception { indexRequest.setJsonEntity(vectors[i]); assertOK(client().performRequest(indexRequest)); } + // always refresh to ensure the data is visible + refresh(indexName); } private static Map search(Request request) throws IOException { diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/connector.sync_job_claim.json b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.sync_job_claim.json new file mode 100644 index 0000000000000..f8d090264038a --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/connector.sync_job_claim.json @@ -0,0 +1,38 @@ +{ + "connector.sync_job_claim": { + "documentation": { + "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/claim-connector-sync-job-api.html", + "description": "Claims a connector sync job." + }, + "stability": "experimental", + "visibility": "public", + "headers": { + "accept": [ + "application/json" + ], + "content_type": [ + "application/json" + ] + }, + "url": { + "paths": [ + { + "path": "/_connector/_sync_job/{connector_sync_job_id}/_claim", + "methods": [ + "PUT" + ], + "parts": { + "connector_sync_job_id": { + "type": "string", + "description": "The unique identifier of the connector sync job to be claimed." + } + } + } + ] + }, + "body": { + "description": "Data to claim a sync job.", + "required": true + } + } +} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json index 5cdb9765ef597..74a6a0a76eda6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json @@ -33,6 +33,11 @@ "master_timeout":{ "type":"time", "description":"Explicit operation timeout for connection to master node" + }, + "wait_for_completion":{ + "type":"boolean", + "description":"Should this request wait until the operation has completed before returning", + "default":true } } } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml index b4147bcfc676e..511ff63d2095d 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cat.shards/10_basic.yml @@ -1,98 +1,99 @@ --- "Help": - requires: - cluster_features: ["gte_v8.11.0"] - reason: dataset size was added in 8.11.0 + cluster_features: [ "gte_v8.15.0" ] + reason: sparse vector was added in 8.15.0 - do: cat.shards: help: true - match: $body: | - /^ index .+ \n - shard .+ \n - prirep .+ \n - state .+ \n - docs .+ \n - store .+ \n - dataset .+ \n - ip .+ \n - id .+ \n - node .+ \n - sync_id .+ \n - unassigned.reason .+ \n - unassigned.at .+ \n - unassigned.for .+ \n - unassigned.details .+ \n - recoverysource.type .+ \n - completion.size .+ \n - fielddata.memory_size .+ \n - fielddata.evictions .+ \n - query_cache.memory_size .+ \n - query_cache.evictions .+ \n - flush.total .+ \n - flush.total_time .+ \n - get.current .+ \n - get.time .+ \n - get.total .+ \n - get.exists_time .+ \n - get.exists_total .+ \n - get.missing_time .+ \n - get.missing_total .+ \n - indexing.delete_current .+ \n - indexing.delete_time .+ \n - indexing.delete_total .+ \n - indexing.index_current .+ \n - indexing.index_time .+ \n - indexing.index_total .+ \n - indexing.index_failed .+ \n - merges.current .+ \n - merges.current_docs .+ \n - merges.current_size .+ \n - merges.total .+ \n - merges.total_docs .+ \n - merges.total_size .+ \n - merges.total_time .+ \n - refresh.total .+ \n - refresh.time .+ \n - refresh.external_total .+ \n - refresh.external_time .+ \n - refresh.listeners .+ \n - search.fetch_current .+ \n - search.fetch_time .+ \n - search.fetch_total .+ \n - search.open_contexts .+ \n - search.query_current .+ \n - search.query_time .+ \n - search.query_total .+ \n - search.scroll_current .+ \n - search.scroll_time .+ \n - search.scroll_total .+ \n - segments.count .+ \n - segments.memory .+ \n - segments.index_writer_memory .+ \n - segments.version_map_memory .+ \n - segments.fixed_bitset_memory .+ \n - seq_no.max .+ \n - seq_no.local_checkpoint .+ \n - seq_no.global_checkpoint .+ \n - warmer.current .+ \n - warmer.total .+ \n - warmer.total_time .+ \n - path.data .+ \n - path.state .+ \n - bulk.total_operations .+ \n - bulk.total_time .+ \n - bulk.total_size_in_bytes .+ \n - bulk.avg_time .+ \n - bulk.avg_size_in_bytes .+ \n - dense_vector.value_count .+ \n - $/ + /^ index .+ \n + shard .+ \n + prirep .+ \n + state .+ \n + docs .+ \n + store .+ \n + dataset .+ \n + ip .+ \n + id .+ \n + node .+ \n + sync_id .+ \n + unassigned.reason .+ \n + unassigned.at .+ \n + unassigned.for .+ \n + unassigned.details .+ \n + recoverysource.type .+ \n + completion.size .+ \n + fielddata.memory_size .+ \n + fielddata.evictions .+ \n + query_cache.memory_size .+ \n + query_cache.evictions .+ \n + flush.total .+ \n + flush.total_time .+ \n + get.current .+ \n + get.time .+ \n + get.total .+ \n + get.exists_time .+ \n + get.exists_total .+ \n + get.missing_time .+ \n + get.missing_total .+ \n + indexing.delete_current .+ \n + indexing.delete_time .+ \n + indexing.delete_total .+ \n + indexing.index_current .+ \n + indexing.index_time .+ \n + indexing.index_total .+ \n + indexing.index_failed .+ \n + merges.current .+ \n + merges.current_docs .+ \n + merges.current_size .+ \n + merges.total .+ \n + merges.total_docs .+ \n + merges.total_size .+ \n + merges.total_time .+ \n + refresh.total .+ \n + refresh.time .+ \n + refresh.external_total .+ \n + refresh.external_time .+ \n + refresh.listeners .+ \n + search.fetch_current .+ \n + search.fetch_time .+ \n + search.fetch_total .+ \n + search.open_contexts .+ \n + search.query_current .+ \n + search.query_time .+ \n + search.query_total .+ \n + search.scroll_current .+ \n + search.scroll_time .+ \n + search.scroll_total .+ \n + segments.count .+ \n + segments.memory .+ \n + segments.index_writer_memory .+ \n + segments.version_map_memory .+ \n + segments.fixed_bitset_memory .+ \n + seq_no.max .+ \n + seq_no.local_checkpoint .+ \n + seq_no.global_checkpoint .+ \n + warmer.current .+ \n + warmer.total .+ \n + warmer.total_time .+ \n + path.data .+ \n + path.state .+ \n + bulk.total_operations .+ \n + bulk.total_time .+ \n + bulk.total_size_in_bytes .+ \n + bulk.avg_time .+ \n + bulk.avg_size_in_bytes .+ \n + dense_vector.value_count .+ \n + sparse_vector.value_count .+ \n + $/ --- "Test cat shards output": - requires: cluster_features: [ "gte_v8.11.0" ] - reason: dataset size was added in 8.11.0 + reason: dataset size was added in 8.11.0 - do: cat.shards: @@ -100,7 +101,7 @@ - match: $body: | - /^$/ + /^$/ - do: indices.create: index: index1 @@ -114,7 +115,7 @@ - match: $body: | - /^(index1 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){10}$/ + /^(index1 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){10}$/ - do: indices.create: @@ -129,14 +130,14 @@ index: i* - match: $body: | - /^(index(1|2) \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){15}$/ + /^(index(1|2) \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){15}$/ - do: cat.shards: index: index2 - match: $body: | - /^(index2 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){5}$/ + /^(index2 \s+ \d \s+ (p|r) \s+ ((STARTED|INITIALIZING|RELOCATING) \s+ (\d \s+ (\d+|\d+[.]\d+)(kb|b) \s+ (\d+|\d+[.]\d+)(kb|b) \s+)? \d{1,3}.\d{1,3}.\d{1,3}.\d{1,3} \s+ .+|UNASSIGNED \s+) \n?){5}$/ --- "Test cat shards using wildcards": @@ -173,7 +174,7 @@ - match: $body: | - /^(foo \n?)$/ + /^(foo \n?)$/ - do: cat.shards: @@ -183,13 +184,13 @@ - match: $body: | - /^(ba(r|z) \n?){2}$/ + /^(ba(r|z) \n?){2}$/ --- "Test cat shards sort": - requires: cluster_features: [ "gte_v8.11.0" ] - reason: dataset size was added in 8.11.0 + reason: dataset size was added in 8.11.0 - do: indices.create: @@ -215,34 +216,34 @@ - do: cat.shards: - h: [index, docs] - s: [docs] + h: [ index, docs ] + s: [ docs ] index: "foo,bar" -# don't use the store here it's cached and might be stale + # don't use the store here it's cached and might be stale - match: $body: | - /^ foo \s+ 0\n - bar \s+ 1\n - $/ + /^ foo \s+ 0\n + bar \s+ 1\n + $/ - do: cat.shards: - h: [index, dataset] - s: [docs] + h: [ index, dataset ] + s: [ docs ] index: "foo,bar" - match: $body: | - /^ foo \s+ (\d+|\d+[.]\d+)(kb|b)\n - bar \s+ (\d+|\d+[.]\d+)(kb|b)\n - $/ + /^ foo \s+ (\d+|\d+[.]\d+)(kb|b)\n + bar \s+ (\d+|\d+[.]\d+)(kb|b)\n + $/ --- "Test cat shards with hidden indices": - requires: - cluster_features: ["gte_v8.3.0"] - reason: hidden indices were misreported in versions before 8.3.0 + cluster_features: [ "gte_v8.3.0" ] + reason: hidden indices were misreported in versions before 8.3.0 - do: indices.create: @@ -261,7 +262,7 @@ - do: cat.shards: - h: [index, docs] + h: [ index, docs ] - match: $body: /foo \s+ 1\n/ diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml index b38a03d53f89f..cf43797a451e7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/cluster.stats/10_basic.yml @@ -1,23 +1,23 @@ --- "cluster stats test": - do: - cluster.stats: {} + cluster.stats: { } - is_true: timestamp - is_true: cluster_name - - match: {status: green} - - gte: { indices.count: 0} + - match: { status: green } + - gte: { indices.count: 0 } - is_true: indices.docs - is_true: indices.store - is_true: indices.fielddata - is_true: indices.query_cache - is_true: indices.completion - is_true: indices.segments - - gte: { nodes.count.total: 1} - - gte: { nodes.count.master: 1} - - gte: { nodes.count.data: 1} - - gte: { nodes.count.ingest: 0} - - gte: { nodes.count.coordinating_only: 0} + - gte: { nodes.count.total: 1 } + - gte: { nodes.count.master: 1 } + - gte: { nodes.count.data: 1 } + - gte: { nodes.count.ingest: 0 } + - gte: { nodes.count.coordinating_only: 0 } - is_true: nodes.os - is_true: nodes.os.mem.total_in_bytes - is_true: nodes.os.mem.free_in_bytes @@ -30,28 +30,54 @@ - is_true: nodes.plugins - is_true: nodes.network_types +--- +"cluster stats with human flag returns docs as human readable size": + - requires: + test_runner_features: [ capabilities ] + capabilities: + - method: GET + path: /_cluster/stats + capabilities: + - "human-readable-total-docs-size" + reason: "Capability required to run test" + + - do: + index: + index: test + id: "1" + refresh: true + body: + foo: bar + + - do: + cluster.stats: + human: true + + - exists: indices.docs.total_size_in_bytes + - exists: indices.docs.total_size + --- "get cluster stats returns cluster_uuid at the top level": - do: - cluster.stats: {} + cluster.stats: { } - is_true: cluster_uuid - is_true: timestamp - is_true: cluster_name - - match: {status: green} - - gte: { indices.count: 0} + - match: { status: green } + - gte: { indices.count: 0 } - is_true: indices.docs - is_true: indices.store - is_true: indices.fielddata - is_true: indices.query_cache - is_true: indices.completion - is_true: indices.segments - - gte: { nodes.count.total: 1} - - gte: { nodes.count.master: 1} - - gte: { nodes.count.data: 1} - - gte: { nodes.count.ingest: 0} - - gte: { nodes.count.coordinating_only: 0} + - gte: { nodes.count.total: 1 } + - gte: { nodes.count.master: 1 } + - gte: { nodes.count.data: 1 } + - gte: { nodes.count.ingest: 0 } + - gte: { nodes.count.coordinating_only: 0 } - is_true: nodes.os - is_true: nodes.os.mem.total_in_bytes - is_true: nodes.os.mem.free_in_bytes @@ -68,7 +94,7 @@ "get cluster stats returns discovery types": - do: - cluster.stats: {} + cluster.stats: { } - is_true: nodes.discovery_types @@ -76,31 +102,31 @@ "get cluster stats returns packaging types": - requires: - cluster_features: ["gte_v7.2.0"] - reason: "packaging types are added for v7.2.0" + cluster_features: [ "gte_v7.2.0" ] + reason: "packaging types are added for v7.2.0" - do: - cluster.stats: {} + cluster.stats: { } - is_true: nodes.packaging_types --- "get cluster stats without runtime fields": - requires: - cluster_features: ["gte_v7.13.0"] - reason: "cluster stats includes runtime fields from 7.13 on" + cluster_features: [ "gte_v7.13.0" ] + reason: "cluster stats includes runtime fields from 7.13 on" - do: indices.create: index: sensor - - do: {cluster.stats: {}} + - do: { cluster.stats: { } } - length: { indices.mappings.field_types: 0 } - length: { indices.mappings.runtime_field_types: 0 } --- "Usage stats with script-less runtime fields": - requires: - cluster_features: ["gte_v7.13.0"] - reason: "cluster stats includes runtime fields from 7.13 on" + cluster_features: [ "gte_v7.13.0" ] + reason: "cluster stats includes runtime fields from 7.13 on" - do: indices.create: index: sensor @@ -122,7 +148,7 @@ bad_map: type: long - - do: {cluster.stats: {}} + - do: { cluster.stats: { } } - length: { indices.mappings.field_types: 3 } - match: { indices.mappings.field_types.0.name: keyword } @@ -145,9 +171,9 @@ - match: { indices.mappings.runtime_field_types.0.shadowed_count: 1 } - match: { indices.mappings.runtime_field_types.0.source_max: 0 } - match: { indices.mappings.runtime_field_types.0.source_total: 0 } - - match: { indices.mappings.runtime_field_types.0.lines_max: 0 } + - match: { indices.mappings.runtime_field_types.0.lines_max: 0 } - match: { indices.mappings.runtime_field_types.0.lines_total: 0 } - - match: { indices.mappings.runtime_field_types.0.chars_max: 0 } + - match: { indices.mappings.runtime_field_types.0.chars_max: 0 } - match: { indices.mappings.runtime_field_types.0.chars_total: 0 } - match: { indices.mappings.runtime_field_types.0.doc_max: 0 } - match: { indices.mappings.runtime_field_types.0.doc_total: 0 } @@ -159,9 +185,9 @@ - match: { indices.mappings.runtime_field_types.1.shadowed_count: 1 } - match: { indices.mappings.runtime_field_types.1.source_max: 0 } - match: { indices.mappings.runtime_field_types.1.source_total: 0 } - - match: { indices.mappings.runtime_field_types.1.lines_max: 0 } + - match: { indices.mappings.runtime_field_types.1.lines_max: 0 } - match: { indices.mappings.runtime_field_types.1.lines_total: 0 } - - match: { indices.mappings.runtime_field_types.1.chars_max: 0 } + - match: { indices.mappings.runtime_field_types.1.chars_max: 0 } - match: { indices.mappings.runtime_field_types.1.chars_total: 0 } - match: { indices.mappings.runtime_field_types.1.doc_max: 0 } - match: { indices.mappings.runtime_field_types.1.doc_total: 0 } @@ -169,8 +195,8 @@ --- "mappings sizes reported in get cluster stats": - requires: - cluster_features: ["gte_v8.4.0"] - reason: "mapping sizes reported from 8.4 onwards" + cluster_features: [ "gte_v8.4.0" ] + reason: "mapping sizes reported from 8.4 onwards" - do: indices.create: index: sensor @@ -180,7 +206,7 @@ "field": "type": "keyword" - - do: {cluster.stats: {}} + - do: { cluster.stats: { } } - gt: { indices.mappings.total_field_count: 0 } - gt: { indices.mappings.total_deduplicated_field_count: 0 } - gt: { indices.mappings.total_deduplicated_mapping_size_in_bytes: 0 } @@ -189,8 +215,8 @@ --- "snapshot stats reported in get cluster stats": - requires: - cluster_features: ["gte_v8.8.0"] - reason: "snapshot stats reported from 8.8 onwards" + cluster_features: [ "gte_v8.8.0" ] + reason: "snapshot stats reported from 8.8 onwards" - do: snapshot.create_repository: @@ -232,14 +258,12 @@ --- "Dense vector stats": - requires: - cluster_features: ["gte_v8.10.0"] - reason: "dense vector stats added in 8.10" + cluster_features: [ "gte_v8.15.0" ] + reason: "dense vector stats reports from primary indices in 8.15" - do: indices.create: index: test1 body: - settings: - number_of_replicas: 0 mappings: properties: vector: @@ -257,8 +281,6 @@ indices.create: index: test2 body: - settings: - number_of_replicas: 0 mappings: properties: vector: @@ -305,11 +327,113 @@ another_vector: [ 10, 11, 12 ] - do: - indices.refresh: {} + indices.refresh: { } - - do: {cluster.stats: {}} + - do: { cluster.stats: { } } - match: { indices.docs.count: 4 } - match: { indices.docs.deleted: 0 } - match: { indices.dense_vector.value_count: 8 } +--- +"Sparse vector stats": + - requires: + cluster_features: [ "gte_v8.15.0" ] + reason: "sparse vector stats added in 8.15" + - do: + indices.create: + index: test1 + body: + settings: + number_of_replicas: 0 + mappings: + properties: + vector: + type: sparse_vector + another_vector: + type: sparse_vector + not_a_vector: + type: keyword + + - do: + indices.create: + index: test2 + body: + settings: + number_of_replicas: 0 + mappings: + properties: + vector: + type: sparse_vector + another_vector: + type: sparse_vector + + - do: + index: + index: test1 + id: "1" + body: + vector: + a: 1.0 + b: 2.0 + c: 3.0 + another_vector: + d: 4.0 + e: 5.0 + f: 6.0 + not_a_vector: "I'm not a vector" + + - do: + index: + index: test1 + id: "2" + body: + vector: + g: 7.0 + h: 8.0 + i: 9.0 + another_vector: + j: 10.0 + k: 11.0 + l: 12.0 + + - do: + index: + index: test1 + id: "3" + body: + not_a_vector: "seriously, I'm not a vector" + + - do: + index: + index: test2 + id: "1" + body: + vector: + a: 1.0 + b: 2.0 + c: 3.0 + another_vector: + d: 4.0 + e: 5.0 + f: 6.0 + + - do: + index: + index: test2 + id: "2" + body: + vector: + g: 7.0 + h: 8.0 + i: 9.0 + + - do: + indices.refresh: { } + + - do: { cluster.stats: { } } + + - match: { indices.docs.count: 5 } + - match: { indices.docs.deleted: 0 } + - match: { indices.sparse_vector.value_count: 7 } + diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml index 5d5110fb54e45..46aa0862b7d9b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/health/10_basic.yml @@ -36,10 +36,9 @@ - exists: indicators.shards_availability.details.started_primaries - exists: indicators.shards_availability.details.unassigned_replicas - - match: { indicators.shards_capacity.status: "green" } - - match: { indicators.shards_capacity.symptom: "The cluster has enough room to add new shards." } - - exists: indicators.shards_capacity.details.data.max_shards_in_cluster - - exists: indicators.shards_capacity.details.frozen.max_shards_in_cluster + # The shards_availability indicator is dependent on HealthMetadata being present in the cluster state, which we can't guarantee. + - is_true: indicators.shards_capacity.status + - is_true: indicators.shards_capacity.symptom - is_true: indicators.data_stream_lifecycle.status - is_true: indicators.data_stream_lifecycle.symptom diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml index a763d6e457490..3d95712d30b30 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml @@ -863,18 +863,22 @@ nested object: - '{ "create": { } }' - '{ "name": "aaaa", "nested_field": {"a": 1, "b": 2}, "nested_array": [{ "a": 10, "b": 20 }, { "a": 100, "b": 200 }] }' + - match: { errors: false } + - do: search: index: test - - match: { hits.total.value: 1 } - - match: { hits.hits.0._source.name: aaaa } - - match: { hits.hits.0._source.nested_field.a: 1 } - - match: { hits.hits.0._source.nested_field.b: 2 } - - match: { hits.hits.0._source.nested_array.0.a: 10 } - - match: { hits.hits.0._source.nested_array.0.b: 20 } - - match: { hits.hits.0._source.nested_array.1.a: 100 } - - match: { hits.hits.0._source.nested_array.1.b: 200 } + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.name: aaaa } + - length: { hits.hits.0._source.nested_field: 2 } + - match: { hits.hits.0._source.nested_field.a: 1 } + - match: { hits.hits.0._source.nested_field.b: 2 } + - length: { hits.hits.0._source.nested_array: 2 } + - match: { hits.hits.0._source.nested_array.0.a: 10 } + - match: { hits.hits.0._source.nested_array.0.b: 20 } + - match: { hits.hits.0._source.nested_array.1.a: 100 } + - match: { hits.hits.0._source.nested_array.1.b: 200 } --- @@ -906,15 +910,201 @@ nested object next to regular: - '{ "create": { } }' - '{ "name": "aaaa", "path": { "to": { "nested": [{ "a": 10, "b": 20 }, { "a": 100, "b": 200 } ], "regular": [{ "a": 10, "b": 20 }, { "a": 100, "b": 200 } ] } } }' + - match: { errors: false } + - do: search: index: test - - match: { hits.total.value: 1 } - - match: { hits.hits.0._source.name: aaaa } - - match: { hits.hits.0._source.path.to.nested.0.a: 10 } - - match: { hits.hits.0._source.path.to.nested.0.b: 20 } - - match: { hits.hits.0._source.path.to.nested.1.a: 100 } - - match: { hits.hits.0._source.path.to.nested.1.b: 200 } - - match: { hits.hits.0._source.path.to.regular.a: [ 10, 100 ] } - - match: { hits.hits.0._source.path.to.regular.b: [ 20, 200 ] } + - match: { hits.total.value: 1 } + - match: { hits.hits.0._source.name: aaaa } + - length: { hits.hits.0._source.path.to.nested: 2 } + - match: { hits.hits.0._source.path.to.nested.0.a: 10 } + - match: { hits.hits.0._source.path.to.nested.0.b: 20 } + - match: { hits.hits.0._source.path.to.nested.1.a: 100 } + - match: { hits.hits.0._source.path.to.nested.1.b: 200 } + - match: { hits.hits.0._source.path.to.regular.a: [ 10, 100 ] } + - match: { hits.hits.0._source.path.to.regular.b: [ 20, 200 ] } + + +--- +nested object with disabled: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + obj_field: + properties: + obj1: + enabled: false + sub_nested: + type: nested + nested_field: + type: nested + properties: + obj1: + enabled: false + nested_array: + type: nested + properties: + obj1: + enabled: false + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 0, "nested_field": {"a": 1, "b": 2, "obj1": { "foo": "bar", "k": [1, 2, 3]}}, "nested_array": [{ "a": 10, "b": 20, "obj1": [{"field1": 1, "field2": 2}, {"field3": 3, "field4": 4}]}, { "a": 100, "b": 200, "obj1": {"field5": 5, "field6": 6}}]}' + - '{ "create": { } }' + - '{ "id": 1, "obj_field": {"a": 1, "b": 2, "obj1": { "foo": "bar", "k": [1, 2, 3]}, "sub_nested": [{ "a": 10, "b": 20}, { "a": 100, "b": 200}]}}' + + - match: { errors: false } + + - do: + search: + index: test + sort: "id" + + - match: { hits.total.value: 2 } + - length: { hits.hits.0._source: 3 } + - match: { hits.hits.0._source.id: 0 } + - length: { hits.hits.0._source.nested_field: 3 } + - match: { hits.hits.0._source.nested_field.a: 1 } + - match: { hits.hits.0._source.nested_field.b: 2 } + - length: { hits.hits.0._source.nested_field.obj1: 2 } + - match: { hits.hits.0._source.nested_field.obj1.foo: "bar" } + - match: { hits.hits.0._source.nested_field.obj1.k: [1, 2, 3] } + - length: { hits.hits.0._source.nested_array: 2 } + - match: { hits.hits.0._source.nested_array.0.a: 10 } + - match: { hits.hits.0._source.nested_array.0.b: 20 } + - length: { hits.hits.0._source.nested_array.0.obj1: 2 } + - match: { hits.hits.0._source.nested_array.0.obj1.0.field1: 1 } + - match: { hits.hits.0._source.nested_array.0.obj1.0.field2: 2 } + - match: { hits.hits.0._source.nested_array.0.obj1.1.field3: 3 } + - match: { hits.hits.0._source.nested_array.0.obj1.1.field4: 4 } + - length: { hits.hits.0._source.nested_array.1: 3 } + - match: { hits.hits.0._source.nested_array.1.a: 100 } + - match: { hits.hits.0._source.nested_array.1.b: 200 } + - length: { hits.hits.0._source.nested_array.1.obj1: 2 } + - match: { hits.hits.0._source.nested_array.1.obj1.field5: 5 } + - match: { hits.hits.0._source.nested_array.1.obj1.field6: 6 } + - length: { hits.hits.1._source: 2 } + - match: { hits.hits.1._source.id: 1 } + - length: { hits.hits.1._source.obj_field: 4 } + - match: { hits.hits.1._source.obj_field.a: 1 } + - match: { hits.hits.1._source.obj_field.b: 2 } + - length: { hits.hits.1._source.obj_field.obj1: 2 } + - match: { hits.hits.1._source.obj_field.obj1.foo: "bar" } + - match: { hits.hits.1._source.obj_field.obj1.k: [ 1, 2, 3 ] } + - length: { hits.hits.1._source.obj_field.sub_nested: 2 } + - length: { hits.hits.1._source.obj_field.sub_nested.0: 2 } + - match: { hits.hits.1._source.obj_field.sub_nested.0.a: 10 } + - match: { hits.hits.1._source.obj_field.sub_nested.0.b: 20 } + - length: { hits.hits.1._source.obj_field.sub_nested.1: 2 } + - match: { hits.hits.1._source.obj_field.sub_nested.1.a: 100 } + - match: { hits.hits.1._source.obj_field.sub_nested.1.b: 200 } + + +--- +doubly nested object: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: requires tracking ignored source + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + obj_field: + properties: + obj1: + enabled: false + sub_nested: + type: nested + nested_field: + type: nested + properties: + sub_nested_field: + type: nested + properties: + obj1: + enabled: false + + - do: + bulk: + index: test + refresh: true + body: + - '{ "create": { } }' + - '{ "id": 0, "nested_field": {"a": 1, "b": 2, "sub_nested_field": {"foo": "bar", "k": [1, 2, 3]}}}' + - '{ "create": { } }' + - '{ "id": 1, "nested_field": {"a": 2, "b": 3, "sub_nested_field": [{"foo": "baz", "k": [4, 50, 6]}, {"foo": "bar"}]}}' + - '{ "create": { } }' + - '{ "id": 2, "nested_field": [{"a": 20, "b": 30, "sub_nested_field": [{"foo": "foobar", "k": [7, 8, 9]}, {"k": [400, 500, 6]}]}, {"a": 0, "b": 33, "sub_nested_field": [{"other": "value", "k": [1, 2, -3]}, {"number": 42}]}]}' + - '{ "create": { } }' + - '{ "id": 3}' + + - match: { errors: false } + + - do: + search: + index: test + sort: "id" + + - match: { hits.total.value: 4 } + - length: { hits.hits.0._source: 2 } + - match: { hits.hits.0._source.id: 0 } + - length: { hits.hits.0._source.nested_field: 3 } + - match: { hits.hits.0._source.nested_field.a: 1 } + - match: { hits.hits.0._source.nested_field.b: 2 } + - length: { hits.hits.0._source.nested_field.sub_nested_field: 2 } + - match: { hits.hits.0._source.nested_field.sub_nested_field.foo: "bar" } + - match: { hits.hits.0._source.nested_field.sub_nested_field.k: [ 1, 2, 3 ] } + - length: { hits.hits.1._source: 2 } + - match: { hits.hits.1._source.id: 1 } + - length: { hits.hits.1._source.nested_field: 3 } + - match: { hits.hits.1._source.nested_field.a: 2 } + - match: { hits.hits.1._source.nested_field.b: 3 } + - length: { hits.hits.1._source.nested_field.sub_nested_field: 2 } + - length: { hits.hits.1._source.nested_field.sub_nested_field.0: 2 } + - match: { hits.hits.1._source.nested_field.sub_nested_field.0.foo: "baz" } + - match: { hits.hits.1._source.nested_field.sub_nested_field.0.k: [ 4, 6, 50 ] } + - length: { hits.hits.1._source.nested_field.sub_nested_field.1: 1 } + - match: { hits.hits.1._source.nested_field.sub_nested_field.1.foo: "bar" } + - length: { hits.hits.2._source: 2 } + - match: { hits.hits.2._source.id: 2 } + - length: { hits.hits.2._source.nested_field: 2 } + - length: { hits.hits.2._source.nested_field.0: 3 } + - match: { hits.hits.2._source.nested_field.0.a: 20 } + - match: { hits.hits.2._source.nested_field.0.b: 30 } + - length: { hits.hits.2._source.nested_field.0.sub_nested_field: 2 } + - length: { hits.hits.2._source.nested_field.0.sub_nested_field.0: 2 } + - match: { hits.hits.2._source.nested_field.0.sub_nested_field.0.foo: "foobar" } + - match: { hits.hits.2._source.nested_field.0.sub_nested_field.0.k: [ 7, 8, 9 ] } + - length: { hits.hits.2._source.nested_field.0.sub_nested_field.1: 1 } + - match: { hits.hits.2._source.nested_field.0.sub_nested_field.1.k: [6, 400, 500] } + - length: { hits.hits.2._source.nested_field.1: 3 } + - match: { hits.hits.2._source.nested_field.1.a: 0 } + - match: { hits.hits.2._source.nested_field.1.b: 33 } + - length: { hits.hits.2._source.nested_field.1.sub_nested_field: 2 } + - length: { hits.hits.2._source.nested_field.1.sub_nested_field.0: 2 } + - match: { hits.hits.2._source.nested_field.1.sub_nested_field.0.other: "value" } + - match: { hits.hits.2._source.nested_field.1.sub_nested_field.0.k: [ -3, 1, 2 ] } + - length: { hits.hits.2._source.nested_field.1.sub_nested_field.1: 1 } + - match: { hits.hits.2._source.nested_field.1.sub_nested_field.1.number: 42 } + - length: { hits.hits.3._source: 1 } + - match: { hits.hits.3._source.id: 3 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml index 128903f4faac8..5e8948b7fdea3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml @@ -31,7 +31,7 @@ create logs index: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -48,17 +48,17 @@ create logs index: refresh: true body: - { "index": { } } - - { "@timestamp": "2024-02-12T10:30:00Z", "hostname": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "message": "No, I am your father." } + - { "@timestamp": "2024-02-12T10:30:00Z", ignored_field_stats: "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "message": "No, I am your father." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:31:00Z", "hostname": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "message": "Do. Or do not. There is no try." } + - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "message": "Do. Or do not. There is no try." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:32:00Z", "hostname": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "message": "May the force be with you." } + - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "message": "May the force be with you." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:33:00Z", "hostname": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "message": "I find your lack of faith disturbing." } + - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "message": "I find your lack of faith disturbing." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:34:00Z", "hostname": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "message": "Wars not make one great." } + - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "message": "Wars not make one great." } - { "index": { } } - - { "@timestamp": "2024-02-12T10:35:00Z", "hostname": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "message": "That's no moon. It's a space station." } + - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "message": "That's no moon. It's a space station." } - do: @@ -103,7 +103,7 @@ using default timestamp field mapping: number_of_shards: 2 mappings: properties: - hostname: + host.name: type: keyword agent_id: type: keyword @@ -149,7 +149,7 @@ missing hostname field: - match: { error.root_cause.0.type: "illegal_argument_exception" } - match: { error.type: "illegal_argument_exception" } - - match: { error.reason: "unknown index sort field:[hostname]" } + - match: { error.reason: "unknown index sort field:[host.name]" } --- missing sort field: @@ -177,7 +177,7 @@ missing sort field: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -271,7 +271,7 @@ override sort order settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -319,7 +319,7 @@ override sort missing settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -367,7 +367,7 @@ override sort mode settings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -410,12 +410,12 @@ override sort field using nested field type in sorting: number_of_replicas: 0 number_of_shards: 2 sort: - field: [ "hostname", "nested", "@timestamp" ] + field: [ "host.name", "nested", "@timestamp" ] mappings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -459,7 +459,7 @@ override sort field using nested field type: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -499,12 +499,12 @@ routing path not allowed in logs mode: mode: logs number_of_replicas: 0 number_of_shards: 2 - routing_path: [ "hostname", "agent_id" ] + routing_path: [ "host.name", "agent_id" ] mappings: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -545,7 +545,7 @@ start time not allowed in logs mode: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword @@ -586,7 +586,7 @@ end time not allowed in logs mode: properties: "@timestamp": type: date - hostname: + host.name: type: keyword agent_id: type: keyword diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml index a0abff2d6726f..bd14fb182ac5a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.highlight/60_unified_matched_fields.yml @@ -10,7 +10,7 @@ setup: settings: index: number_of_shards: 1 - number_of_replicas: 0 + number_of_replicas: 1 analysis: filter: my_edge_ngram: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml index 547ff83d91360..b7a5517309949 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_byte_quantized.yml @@ -6,6 +6,9 @@ setup: indices.create: index: hnsw_byte_quantized body: + settings: + index: + number_of_shards: 1 mappings: properties: name: @@ -17,7 +20,6 @@ setup: similarity: l2_norm index_options: type: int8_hnsw - confidence_interval: 0.9 another_vector: type: dense_vector dims: 5 @@ -25,7 +27,6 @@ setup: similarity: l2_norm index_options: type: int8_hnsw - confidence_interval: 0.9 - do: index: @@ -35,10 +36,10 @@ setup: name: cow.jpg vector: [230.0, 300.33, -34.8988, 15.555, -200.0] another_vector: [130.0, 115.0, -1.02, 15.555, -100.0] - # Flush in order to provoke a merge later - do: - indices.flush: { } + indices.flush: + index: hnsw_byte_quantized - do: index: @@ -48,10 +49,10 @@ setup: name: moose.jpg vector: [-0.5, 100.0, -13, 14.8, -156.0] another_vector: [-0.5, 50.0, -1, 1, 120] - # Flush in order to provoke a merge later - do: - indices.flush: { } + indices.flush: + index: hnsw_byte_quantized - do: index: @@ -61,15 +62,15 @@ setup: name: rabbit.jpg vector: [0.5, 111.3, -13.0, 14.8, -156.0] another_vector: [-0.5, 11.0, 0, 12, 111.0] + # Flush in order to provoke a merge later + - do: + indices.flush: + index: hnsw_byte_quantized - do: indices.forcemerge: index: hnsw_byte_quantized max_num_segments: 1 - - - do: - indices.refresh: {} - --- "kNN search only": - do: @@ -96,8 +97,8 @@ setup: body: fields: [ "name" ] knn: - - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} - - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8, -156.0], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12, 111.0], k: 2, num_candidates: 3} - match: {hits.hits.0._id: "3"} - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} @@ -246,6 +247,9 @@ setup: indices.create: index: mip body: + settings: + index: + number_of_shards: 1 mappings: properties: name: @@ -402,6 +406,10 @@ setup: - do: indices.create: index: hnsw_byte_quantized_merge_cosine + body: + settings: + index: + number_of_shards: 1 - do: indices.put_mapping: @@ -475,6 +483,10 @@ setup: - do: indices.create: index: hnsw_byte_quantized_merge_dot_product + body: + settings: + index: + number_of_shards: 1 - do: indices.put_mapping: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml new file mode 100644 index 0000000000000..24437e3db1379 --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/41_knn_search_half_byte_quantized.yml @@ -0,0 +1,591 @@ +setup: + - requires: + cluster_features: "mapper.vectors.int4_quantization" + reason: 'kNN float to half-byte quantization is required' + - do: + indices.create: + index: hnsw_byte_quantized + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 4 + index: true + similarity: l2_norm + index_options: + type: int4_hnsw + another_vector: + type: dense_vector + dims: 4 + index: true + similarity: l2_norm + index_options: + type: int4_hnsw + + - do: + index: + index: hnsw_byte_quantized + id: "1" + body: + name: cow.jpg + vector: [230.0, 300.33, -34.8988, 15.555] + another_vector: [130.0, 115.0, -1.02, 15.555] + + # Flush in order to provoke a merge later + - do: + indices.flush: { } + + - do: + index: + index: hnsw_byte_quantized + id: "2" + body: + name: moose.jpg + vector: [-0.5, 100.0, -13, 14.8] + another_vector: [-0.5, 50.0, -1, 1] + + # Flush in order to provoke a merge later + - do: + indices.flush: { } + + - do: + index: + index: hnsw_byte_quantized + id: "3" + body: + name: rabbit.jpg + vector: [0.5, 111.3, -13.0, 14.8] + another_vector: [-0.5, 11.0, 0, 12] + + - do: + indices.forcemerge: + index: hnsw_byte_quantized + max_num_segments: 1 + + - do: + indices.refresh: {} + +--- +"kNN search only": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 2 + num_candidates: 3 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search only": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12], k: 2, num_candidates: 3} + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} +--- +"kNN search plus query": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 2 + num_candidates: 3 + query: + term: + name: + value: cow.jpg + boost: 1.5 + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search with query": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12], k: 2, num_candidates: 3, boost: 2.0} + query: + term: + name: + value: cow.jpg + boost: 2.0 + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1.fields.name.0: "cow.jpg"} + + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2.fields.name.0: "moose.jpg"} +--- +"kNN search with filter": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 2 + num_candidates: 3 + filter: + term: + name: "rabbit.jpg" + + - match: {hits.total.value: 1} + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 2 + num_candidates: 3 + filter: + - term: + name: "rabbit.jpg" + - term: + _id: 2 + + - match: {hits.total.value: 0} + +--- +"KNN Vector similarity search only": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 10.3 + query_vector: [-0.5, 90.0, -10, 14.8] + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} +--- +"Vector similarity with filter only": + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 11 + query_vector: [-0.5, 90.0, -10, 14.8] + filter: {"term": {"name": "moose.jpg"}} + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - do: + search: + index: hnsw_byte_quantized + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 110 + query_vector: [-0.5, 90.0, -10, 14.8] + filter: {"term": {"name": "cow.jpg"}} + + - length: {hits.hits: 0} +--- +"Knn search with mip": + - do: + indices.create: + index: mip + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 6 + index: true + similarity: max_inner_product + index_options: + type: int4_hnsw + + - do: + index: + index: mip + id: "1" + body: + name: cow.jpg + vector: [1, 2, 3, 4, 5, 0] + + # Flush in order to provoke a merge later + - do: + indices.flush: { } + + - do: + index: + index: mip + id: "2" + body: + name: moose.jpg + vector: [1, 1, 1, 1, 1, 0] + + # Flush in order to provoke a merge later + - do: + indices.flush: { } + + - do: + index: + index: mip + id: "3" + body: + name: rabbit.jpg + vector: [1, 2, 2, 2, 2, 0] + + # We force merge into a single segment to make sure scores are more uniform + # Each segment can have a different quantization error, which can affect scores and mip is especially sensitive to this + - do: + indices.forcemerge: + index: mip + max_num_segments: 1 + + - do: + indices.refresh: {} + + - do: + search: + index: mip + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + query_vector: [1, 2, 3, 4, 5, 0] + + + - length: {hits.hits: 3} + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.2._id: "2"} + + - do: + search: + index: mip + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + query_vector: [1, 2, 3, 4, 5, 0] + filter: { "term": { "name": "moose.jpg" } } + + + + - length: {hits.hits: 1} + - match: {hits.hits.0._id: "2"} +--- +"Cosine similarity with indexed vector": + - skip: + features: "headers" + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "cosineSimilarity(params.query_vector, 'vector')" + params: + query_vector: [0.5, 111.3, -13.0, 14.8] + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "3"} + - gte: {hits.hits.0._score: 0.999} + - lte: {hits.hits.0._score: 1.001} + + - match: {hits.hits.1._id: "2"} + - gte: {hits.hits.1._score: 0.998} + - lte: {hits.hits.1._score: 1.0} + + - match: {hits.hits.2._id: "1"} + - gte: {hits.hits.2._score: 0.78} + - lte: {hits.hits.2._score: 0.80} +--- +"Test bad quantization parameters": + - do: + catch: bad_request + indices.create: + index: bad_hnsw_quantized + body: + mappings: + properties: + vector: + type: dense_vector + dims: 6 + element_type: byte + index: true + index_options: + type: int4_hnsw + + - do: + catch: bad_request + indices.create: + index: bad_hnsw_quantized + body: + mappings: + properties: + vector: + type: dense_vector + dims: 6 + index: false + index_options: + type: int4_hnsw +--- +"Test create, merge, and search cosine": + - do: + indices.create: + index: hnsw_byte_quantized_merge_cosine + body: + settings: + index: + number_of_shards: 1 + - do: + indices.put_mapping: + index: hnsw_byte_quantized_merge_cosine + body: + properties: + embedding: + type: dense_vector + element_type: float + similarity: cosine + index_options: + type: int4_hnsw + + - do: + index: + index: hnsw_byte_quantized_merge_cosine + id: "1" + body: + embedding: [1.0, 1.0, 1.0, 1.0] + + # Flush in order to provoke a merge later + - do: + indices.flush: { } + + - do: + index: + index: hnsw_byte_quantized_merge_cosine + id: "2" + body: + embedding: [1.0, 1.0, 1.0, 2.0] + + # Flush in order to provoke a merge later + - do: + indices.flush: { } + + - do: + index: + index: hnsw_byte_quantized_merge_cosine + id: "3" + body: + embedding: [1.0, 1.0, 1.0, 3.0] + + - do: + indices.forcemerge: + index: hnsw_byte_quantized_merge_cosine + max_num_segments: 1 + + - do: + indices.refresh: {} + + - do: + search: + index: hnsw_byte_quantized_merge_cosine + body: + size: 3 + query: + knn: + field: embedding + query_vector: [1.0, 1.0, 1.0, 1.0] + num_candidates: 10 + + - length: { hits.hits: 3 } + - match: { hits.hits.0._id: "1"} + - match: { hits.hits.1._id: "2"} + - match: { hits.hits.2._id: "3"} +--- +"Test create, merge, and search dot_product": + - do: + indices.create: + index: hnsw_byte_quantized_merge_dot_product + body: + settings: + index: + number_of_shards: 1 + - do: + indices.put_mapping: + index: hnsw_byte_quantized_merge_dot_product + body: + properties: + embedding: + type: dense_vector + element_type: float + similarity: dot_product + index_options: + type: int4_hnsw + + - do: + index: + index: hnsw_byte_quantized_merge_dot_product + id: "1" + body: + embedding: [0.6, 0.8] + + # Flush in order to provoke a merge later + - do: + indices.flush: { } + + - do: + index: + index: hnsw_byte_quantized_merge_dot_product + id: "2" + body: + embedding: [0.8, 0.6] + + # Flush in order to provoke a merge later + - do: + indices.flush: { } + + - do: + index: + index: hnsw_byte_quantized_merge_dot_product + id: "3" + body: + embedding: [-0.6, -0.8] + + - do: + indices.forcemerge: + index: hnsw_byte_quantized_merge_dot_product + max_num_segments: 1 + + - do: + indices.refresh: {} + + - do: + search: + index: hnsw_byte_quantized_merge_dot_product + body: + size: 3 + query: + knn: + field: embedding + query_vector: [0.6, 0.8] + num_candidates: 10 + + - length: { hits.hits: 3 } + - match: { hits.hits.0._id: "1"} + - match: { hits.hits.1._id: "2"} + - match: { hits.hits.2._id: "3"} +--- +"Test odd dimensions fail indexing": + - do: + catch: bad_request + indices.create: + index: bad_hnsw_quantized + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + index: true + index_options: + type: int4_hnsw + + - do: + indices.create: + index: dynamic_dim_hnsw_quantized + body: + mappings: + properties: + vector: + type: dense_vector + index: true + similarity: l2_norm + index_options: + type: int4_hnsw + + - do: + catch: bad_request + index: + index: dynamic_dim_hnsw_quantized + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0] + + - do: + index: + index: dynamic_dim_hnsw_quantized + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml new file mode 100644 index 0000000000000..b9a0b16f2bd7a --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int4_flat.yml @@ -0,0 +1,346 @@ +setup: + - requires: + cluster_features: "mapper.vectors.int4_quantization" + reason: 'kNN float to half-byte quantization is required' + - do: + indices.create: + index: int4_flat + body: + settings: + index: + number_of_shards: 1 + mappings: + properties: + name: + type: keyword + vector: + type: dense_vector + dims: 4 + index: true + similarity: l2_norm + index_options: + type: int4_flat + another_vector: + type: dense_vector + dims: 4 + index: true + similarity: l2_norm + index_options: + type: int4_flat + + - do: + index: + index: int4_flat + id: "1" + body: + name: cow.jpg + vector: [230.0, 300.33, -34.8988, 15.555] + another_vector: [130.0, 115.0, -1.02, 15.555] + # Flush in order to provoke a merge later & ensure replicas have same doc order + - do: + indices.flush: { } + - do: + index: + index: int4_flat + id: "2" + body: + name: moose.jpg + vector: [-0.5, 100.0, -13, 14.8] + another_vector: [-0.5, 50.0, -1, 1] + # Flush in order to provoke a merge later & ensure replicas have same doc order + - do: + indices.flush: { } + - do: + index: + index: int4_flat + id: "3" + body: + name: rabbit.jpg + vector: [0.5, 111.3, -13.0, 14.8] + another_vector: [-0.5, 11.0, 0, 12] + + - do: + indices.refresh: {} + +--- +"kNN search only": + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 2 + num_candidates: 3 + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - match: {hits.hits.1._id: "3"} + - match: {hits.hits.1.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search only": + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12], k: 2, num_candidates: 3} + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} +--- +"kNN search plus query": + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 2 + num_candidates: 3 + query: + term: + name: + value: cow.jpg + boost: 1.5 + + - match: {hits.hits.0._id: "1"} + - match: {hits.hits.0.fields.name.0: "cow.jpg"} + + - match: {hits.hits.1._id: "2"} + - match: {hits.hits.1.fields.name.0: "moose.jpg"} + + - match: {hits.hits.2._id: "3"} + - match: {hits.hits.2.fields.name.0: "rabbit.jpg"} +--- +"kNN multi-field search with query": + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + - {field: vector, query_vector: [-0.5, 90.0, -10, 14.8], k: 2, num_candidates: 3} + - {field: another_vector, query_vector: [-0.5, 11.0, 0, 12], k: 2, num_candidates: 3, boost: 2.0} + query: + term: + name: + value: cow.jpg + boost: 2.0 + + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - match: {hits.hits.1._id: "1"} + - match: {hits.hits.1.fields.name.0: "cow.jpg"} + + - match: {hits.hits.2._id: "2"} + - match: {hits.hits.2.fields.name.0: "moose.jpg"} +--- +"kNN search with filter": + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 2 + num_candidates: 3 + filter: + term: + name: "rabbit.jpg" + + - match: {hits.total.value: 1} + - match: {hits.hits.0._id: "3"} + - match: {hits.hits.0.fields.name.0: "rabbit.jpg"} + + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + field: vector + query_vector: [-0.5, 90.0, -10, 14.8] + k: 2 + num_candidates: 3 + filter: + - term: + name: "rabbit.jpg" + - term: + _id: 2 + + - match: {hits.total.value: 0} + +--- +"KNN Vector similarity search only": + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 10.3 + query_vector: [-0.5, 90.0, -10, 14.8] + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} +--- +"Vector similarity with filter only": + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 11 + query_vector: [-0.5, 90.0, -10, 14.8] + filter: {"term": {"name": "moose.jpg"}} + + - length: {hits.hits: 1} + + - match: {hits.hits.0._id: "2"} + - match: {hits.hits.0.fields.name.0: "moose.jpg"} + + - do: + search: + index: int4_flat + body: + fields: [ "name" ] + knn: + num_candidates: 3 + k: 3 + field: vector + similarity: 110 + query_vector: [-0.5, 90.0, -10, 14.8] + filter: {"term": {"name": "cow.jpg"}} + + - length: {hits.hits: 0} +--- +"Cosine similarity with indexed vector": + - skip: + features: "headers" + - do: + headers: + Content-Type: application/json + search: + rest_total_hits_as_int: true + body: + query: + script_score: + query: {match_all: {} } + script: + source: "cosineSimilarity(params.query_vector, 'vector')" + params: + query_vector: [0.5, 111.3, -13.0, 14.8] + + - match: {hits.total: 3} + + - match: {hits.hits.0._id: "3"} + - gte: {hits.hits.0._score: 0.999} + - lte: {hits.hits.0._score: 1.001} + + - match: {hits.hits.1._id: "2"} + - gte: {hits.hits.1._score: 0.998} + - lte: {hits.hits.1._score: 1.0} + + - match: {hits.hits.2._id: "1"} + - gte: {hits.hits.2._score: 0.78} + - lte: {hits.hits.2._score: 0.80} +--- +"Test bad parameters": + - do: + catch: bad_request + indices.create: + index: bad_int4_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 6 + index: true + index_options: + type: int4_flat + m: 42 + + - do: + catch: bad_request + indices.create: + index: bad_int4_flat + body: + mappings: + properties: + vector: + type: dense_vector + dims: 6 + element_type: byte + index: true + index_options: + type: int4_flat +--- +"Test odd dimensions fail indexing": + # verify index creation fails + - do: + catch: bad_request + indices.create: + index: bad_hnsw_quantized + body: + mappings: + properties: + vector: + type: dense_vector + dims: 5 + index: true + similarity: l2_norm + index_options: + type: int4_flat + + # verify dynamic dimension fails + - do: + indices.create: + index: dynamic_dim_hnsw_quantized + body: + mappings: + properties: + vector: + type: dense_vector + index: true + similarity: l2_norm + index_options: + type: int4_hnsw + + # verify index fails for odd dim vector + - do: + catch: bad_request + index: + index: dynamic_dim_hnsw_quantized + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0] + + # verify that we can index an even dim vector after the odd dim vector failure + - do: + index: + index: dynamic_dim_hnsw_quantized + body: + vector: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml index eb08cc472c798..139747c5e7ee5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search.vectors/42_knn_search_int8_flat.yml @@ -6,6 +6,9 @@ setup: indices.create: index: int8_flat body: + settings: + index: + number_of_shards: 1 mappings: properties: name: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.delete/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.delete/10_basic.yml new file mode 100644 index 0000000000000..5a60f76f6da2c --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/snapshot.delete/10_basic.yml @@ -0,0 +1,70 @@ +--- +setup: + + - do: + snapshot.create_repository: + repository: test_repo_create_1 + body: + type: fs + settings: + location: "test_repo_create_1_loc" + + - do: + indices.create: + index: test_index + body: + settings: + number_of_shards: 1 + number_of_replicas: 1 + + - do: + snapshot.create: + repository: test_repo_create_1 + snapshot: test_snapshot + wait_for_completion: true + +--- +"Delete a snapshot synchronously (default)": + + - do: + snapshot.delete: + repository: test_repo_create_1 + snapshot: test_snapshot + + - match: { acknowledged: true } + +--- +"Delete a snapshot synchronously (specified)": + - requires: + test_runner_features: capabilities + capabilities: + - method: DELETE + path: /_snapshot/{repository}/{snapshot} + parameters: [ wait_for_completion ] + reason: "wait_for_completion parameter was introduced in 8.15" + + - do: + snapshot.delete: + repository: test_repo_create_1 + snapshot: test_snapshot + wait_for_completion: true + + - match: { acknowledged: true } + +--- +"Delete a snapshot asynchronously": + - requires: + test_runner_features: capabilities + capabilities: + - method: DELETE + path: /_snapshot/{repository}/{snapshot} + parameters: [ wait_for_completion ] + reason: "wait_for_completion parameter was introduced in 8.15" + + - do: + snapshot.delete: + repository: test_repo_create_1 + snapshot: test_snapshot + wait_for_completion: false + + - match: { acknowledged: true } diff --git a/server/build.gradle b/server/build.gradle index 82bb972a2c173..48a4febfc6cdf 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -37,7 +37,7 @@ dependencies { api project(":libs:elasticsearch-plugin-analysis-api") api project(':libs:elasticsearch-grok') api project(":libs:elasticsearch-tdigest") - implementation project(":libs:elasticsearch-vec") + implementation project(":libs:elasticsearch-simdvec") implementation project(':libs:elasticsearch-plugin-classloader') // no compile dependency by server, but server defines security policy for this codebase so it i> diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java index b0238922c206e..95a5ca9157f49 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/HotThreadsIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.admin.cluster.node.hotthreads.TransportNodesHotThreadsAction; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.ReferenceDocs; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.ChunkedLoggingStreamTestUtils; import org.elasticsearch.core.TimeValue; import org.elasticsearch.monitor.jvm.HotThreads; @@ -48,18 +49,23 @@ public void testHotThreadsDontFail() throws InterruptedException { final int iters = scaledRandomIntBetween(2, 20); final AtomicBoolean hasErrors = new AtomicBoolean(false); for (int i = 0; i < iters; i++) { - final NodesHotThreadsRequest request = new NodesHotThreadsRequest(); - if (randomBoolean()) { - TimeValue timeValue = new TimeValue(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(20, 500)); - request.interval(timeValue); - } - if (randomBoolean()) { - request.threads(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(1, 500)); - } - request.ignoreIdleThreads(randomBoolean()); - if (randomBoolean()) { - request.type(HotThreads.ReportType.of(randomFrom("block", "mem", "cpu", "wait"))); - } + final NodesHotThreadsRequest request = new NodesHotThreadsRequest( + Strings.EMPTY_ARRAY, + new HotThreads.RequestOptions( + randomBoolean() ? HotThreads.RequestOptions.DEFAULT.threads() + : rarely() ? randomIntBetween(500, 5000) + : randomIntBetween(1, 500), + randomBoolean() + ? HotThreads.RequestOptions.DEFAULT.reportType() + : HotThreads.ReportType.of(randomFrom("block", "mem", "cpu", "wait")), + HotThreads.RequestOptions.DEFAULT.sortOrder(), + randomBoolean() + ? HotThreads.RequestOptions.DEFAULT.interval() + : TimeValue.timeValueMillis(rarely() ? randomIntBetween(500, 5000) : randomIntBetween(20, 500)), + HotThreads.RequestOptions.DEFAULT.snapshots(), + randomBoolean() + ) + ); final CountDownLatch latch = new CountDownLatch(1); client().execute(TransportNodesHotThreadsAction.TYPE, request, new ActionListener<>() { @Override @@ -125,7 +131,17 @@ public void testIgnoreIdleThreads() { SubscribableListener.newForked( l -> client().execute( TransportNodesHotThreadsAction.TYPE, - new NodesHotThreadsRequest().ignoreIdleThreads(false).threads(Integer.MAX_VALUE), + new NodesHotThreadsRequest( + Strings.EMPTY_ARRAY, + new HotThreads.RequestOptions( + Integer.MAX_VALUE, + HotThreads.RequestOptions.DEFAULT.reportType(), + HotThreads.RequestOptions.DEFAULT.sortOrder(), + HotThreads.RequestOptions.DEFAULT.interval(), + HotThreads.RequestOptions.DEFAULT.snapshots(), + false + ) + ), l.map(response -> { int length = 0; for (NodeHotThreads node : response.getNodesMap().values()) { @@ -139,7 +155,17 @@ public void testIgnoreIdleThreads() { ); // Second time, do ignore idle threads: - final var request = new NodesHotThreadsRequest().threads(Integer.MAX_VALUE); + final var request = new NodesHotThreadsRequest( + Strings.EMPTY_ARRAY, + new HotThreads.RequestOptions( + Integer.MAX_VALUE, + HotThreads.RequestOptions.DEFAULT.reportType(), + HotThreads.RequestOptions.DEFAULT.sortOrder(), + HotThreads.RequestOptions.DEFAULT.interval(), + HotThreads.RequestOptions.DEFAULT.snapshots(), + HotThreads.RequestOptions.DEFAULT.ignoreIdleThreads() + ) + ); // Make sure default is true: assertTrue(request.ignoreIdleThreads()); final var totSizeIgnoreIdle = safeAwait( @@ -160,26 +186,30 @@ public void testIgnoreIdleThreads() { public void testTimestampAndParams() { safeAwait( SubscribableListener.newForked( - l -> client().execute(TransportNodesHotThreadsAction.TYPE, new NodesHotThreadsRequest(), l.map(response -> { - if (Constants.FREE_BSD) { - for (NodeHotThreads node : response.getNodesMap().values()) { - assertThat(node.getHotThreads(), containsString("hot_threads is not supported")); - } - } else { - for (NodeHotThreads node : response.getNodesMap().values()) { - assertThat( - node.getHotThreads(), - allOf( - containsString("Hot threads at"), - containsString("interval=500ms"), - containsString("busiestThreads=3"), - containsString("ignoreIdleThreads=true") - ) - ); + l -> client().execute( + TransportNodesHotThreadsAction.TYPE, + new NodesHotThreadsRequest(Strings.EMPTY_ARRAY, HotThreads.RequestOptions.DEFAULT), + l.map(response -> { + if (Constants.FREE_BSD) { + for (NodeHotThreads node : response.getNodesMap().values()) { + assertThat(node.getHotThreads(), containsString("hot_threads is not supported")); + } + } else { + for (NodeHotThreads node : response.getNodesMap().values()) { + assertThat( + node.getHotThreads(), + allOf( + containsString("Hot threads at"), + containsString("interval=500ms"), + containsString("busiestThreads=3"), + containsString("ignoreIdleThreads=true") + ) + ); + } } - } - return null; - })) + return null; + }) + ) ) ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java index 954ef3d6d7887..5d4a922ec3e11 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/ReloadSecureSettingsIT.java @@ -65,9 +65,8 @@ private static void executeReloadSecureSettings( SecureString password, ActionListener listener ) { - final var request = new NodesReloadSecureSettingsRequest(); + final var request = new NodesReloadSecureSettingsRequest(nodeIds); try { - request.nodesIds(nodeIds); request.setSecureStorePassword(password); client().execute(TransportNodesReloadSecureSettingsAction.TYPE, request, listener); } finally { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java index 5ea1b869f417e..4ad2a56d2e979 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/cluster/node/tasks/TasksIT.java @@ -510,6 +510,9 @@ public void testTasksCancellation() throws Exception { expectThrows(TaskCancelledException.class, future); + logger.info("--> waiting for all ongoing tasks to complete within a reasonable time"); + safeGet(clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name() + "*").setWaitForCompletion(true).execute()); + logger.info("--> checking that test tasks are not running"); assertEquals(0, clusterAdmin().prepareListTasks().setActions(TEST_TASK_ACTION.name() + "*").get().getTasks().size()); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java index ea566c90ad769..3ff7e66d25639 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/PrevalidateShardPathIT.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.junit.annotations.TestLogging; import java.util.HashSet; import java.util.List; @@ -39,6 +40,10 @@ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) public class PrevalidateShardPathIT extends ESIntegTestCase { + @TestLogging( + value = "org.elasticsearch.cluster.service.MasterService:DEBUG", + reason = "https://github.com/elastic/elasticsearch/issues/104807" + ) public void testCheckShards() throws Exception { internalCluster().startMasterOnlyNode(); String node1 = internalCluster().startDataOnlyNode(); @@ -95,7 +100,6 @@ public void testCheckShards() throws Exception { .allShards() .filter(s -> s.getIndexName().equals(indexName)) .filter(s -> node2ShardIds.contains(s.shardId())) - .filter(s -> s.currentNodeId().equals(node2Id)) .toList(); logger.info("Found {} shards on the relocation source node {} in the cluster state", node2Shards, node2Id); for (var node2Shard : node2Shards) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetIT.java b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetIT.java new file mode 100644 index 0000000000000..671c308f98fbb --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetIT.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.shard.IndexEventListener; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.ESIntegTestCase.ClusterScope; +import org.elasticsearch.test.ESIntegTestCase.Scope; +import org.elasticsearch.test.MockIndexEventListener; + +import java.util.List; + +@ClusterScope(scope = Scope.TEST, numDataNodes = 0) +public class AllocationFailuresResetIT extends ESIntegTestCase { + + private static final String INDEX = "index-1"; + private static final int SHARD = 0; + + @Override + protected List> nodePlugins() { + return List.of(MockIndexEventListener.TestPlugin.class); + } + + private void injectAllocationFailures(String node) { + internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node).setNewDelegate(new IndexEventListener() { + @Override + public void beforeIndexShardCreated(ShardRouting routing, Settings indexSettings) { + throw new RuntimeException("shard allocation failure"); + } + }); + } + + private void removeAllocationFailuresInjection(String node) { + internalCluster().getInstance(MockIndexEventListener.TestEventListener.class, node).setNewDelegate(new IndexEventListener() { + }); + } + + private void awaitShardAllocMaxRetries() throws Exception { + var maxRetries = MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY.get(internalCluster().getDefaultSettings()); + assertBusy(() -> { + var state = clusterAdmin().prepareState().get().getState(); + var index = state.getRoutingTable().index(INDEX); + assertNotNull(index); + var shard = index.shard(SHARD).primaryShard(); + assertNotNull(shard); + var unassigned = shard.unassignedInfo(); + assertNotNull(unassigned); + assertEquals(maxRetries.intValue(), unassigned.failedAllocations()); + }); + } + + private void awaitShardAllocSucceed() throws Exception { + assertBusy(() -> { + var state = clusterAdmin().prepareState().get().getState(); + var index = state.getRoutingTable().index(INDEX); + assertNotNull(index); + var shard = index.shard(SHARD).primaryShard(); + assertNotNull(shard); + assertTrue(shard.assignedToNode()); + assertTrue(shard.started()); + }); + } + + public void testResetFailuresOnNodeJoin() throws Exception { + var node1 = internalCluster().startNode(); + injectAllocationFailures(node1); + prepareCreate(INDEX, indexSettings(1, 0)).execute(); + awaitShardAllocMaxRetries(); + removeAllocationFailuresInjection(node1); + internalCluster().startNode(); + awaitShardAllocSucceed(); + } + +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java index 4bd56e2276d18..2d17e26fd2959 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/common/network/ThreadWatchdogIT.java @@ -35,7 +35,7 @@ import org.elasticsearch.test.MockLog; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.test.transport.MockTransportService; -import org.elasticsearch.transport.TransportRequest; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportService; @@ -137,7 +137,7 @@ public void testThreadWatchdogTransportLogging() { targetTransportService.registerRequestHandler( "internal:slow", EsExecutors.DIRECT_EXECUTOR_SERVICE, - TransportRequest.Empty::new, + EmptyRequest::new, (request, channel, task) -> { blockAndWaitForWatchdogLogs(); channel.sendResponse(TransportResponse.Empty.INSTANCE); @@ -149,7 +149,7 @@ public void testThreadWatchdogTransportLogging() { l -> sourceTransportService.sendRequest( targetTransportService.getLocalNode(), "internal:slow", - new TransportRequest.Empty(), + new EmptyRequest(), new ActionListenerResponseHandler( l, in -> TransportResponse.Empty.INSTANCE, diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java index 1c715beb04356..7e057c19ea82e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/IndexingMemoryControllerIT.java @@ -81,7 +81,8 @@ EngineConfig engineConfigWithLargerIndexingMemory(EngineConfig config) { config.getLeafSorter(), config.getRelativeTimeInNanosSupplier(), config.getIndexCommitListener(), - config.isPromotableToPrimary() + config.isPromotableToPrimary(), + config.getMapperService() ); } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java index 14b13addac84f..4f15b82ca1f16 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/recovery/IndexRecoveryIT.java @@ -322,8 +322,7 @@ public void startShardRecovery(String sourceNode, String targetNode) throws Exce * @param isRecoveryThrottlingNode whether to expect throttling to have occurred on the node */ public void assertNodeHasThrottleTimeAndNoRecoveries(String nodeName, Boolean isRecoveryThrottlingNode) { - NodesStatsResponse nodesStatsResponse = clusterAdmin().prepareNodesStats() - .setNodesIds(nodeName) + NodesStatsResponse nodesStatsResponse = clusterAdmin().prepareNodesStats(nodeName) .clear() .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)) .get(); @@ -612,8 +611,7 @@ public void testRerouteRecovery() throws Exception { validateIndexRecoveryState(recoveryStates.get(0).getIndex()); Consumer assertNodeHasThrottleTimeAndNoRecoveries = nodeName -> { - NodesStatsResponse nodesStatsResponse = clusterAdmin().prepareNodesStats() - .setNodesIds(nodeName) + NodesStatsResponse nodesStatsResponse = clusterAdmin().prepareNodesStats(nodeName) .clear() .setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)) .get(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java index e70c48ce8184e..7ffc2539d2fa0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/stats/IndexStatsIT.java @@ -834,7 +834,8 @@ public void testFlagOrdinalOrder() { Flag.Bulk, Flag.Shards, Flag.Mappings, - Flag.DenseVector }; + Flag.DenseVector, + Flag.SparseVector }; assertThat(flags.length, equalTo(Flag.values().length)); for (int i = 0; i < flags.length; i++) { @@ -1000,6 +1001,7 @@ private static void set(Flag flag, IndicesStatsRequestBuilder builder, boolean s // We don't actually expose shards in IndexStats, but this test fails if it isn't handled builder.request().flags().set(Flag.Shards, set); case DenseVector -> builder.setDenseVector(set); + case SparseVector -> builder.setSparseVector(set); default -> fail("new flag? " + flag); } } @@ -1046,6 +1048,8 @@ private static boolean isSet(Flag flag, CommonStats response) { return response.getNodeMappings() != null; case DenseVector: return response.getDenseVectorStats() != null; + case SparseVector: + return response.getSparseVectorStats() != null; default: fail("new flag? " + flag); return false; diff --git a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java index bf6c59a4c0a9b..58d1d7d88ec55 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/plugins/internal/DocumentSizeObserverIT.java @@ -9,10 +9,10 @@ package org.elasticsearch.plugins.internal; import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.EngineFactory; import org.elasticsearch.index.engine.InternalEngine; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.plugins.EnginePlugin; import org.elasticsearch.plugins.IngestPlugin; @@ -104,7 +104,7 @@ public IndexResult index(Index index) throws IOException { DocumentSizeReporter documentParsingReporter = documentParsingProvider.newDocumentSizeReporter( shardId.getIndexName(), - IndexMode.STANDARD, + config().getMapperService(), DocumentSizeAccumulator.EMPTY_INSTANCE ); documentParsingReporter.onIndexingCompleted(index.parsedDoc()); @@ -136,7 +136,7 @@ public DocumentSizeObserver newDocumentSizeObserver() { @Override public DocumentSizeReporter newDocumentSizeReporter( String indexName, - IndexMode indexMode, + MapperService mapperService, DocumentSizeAccumulator documentSizeAccumulator ) { return new TestDocumentSizeReporter(indexName); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java index b7a1dc12406d2..5b44a949ab784 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/readiness/ReadinessClusterIT.java @@ -122,6 +122,7 @@ private void expectMasterNotFound() { ); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/108613") public void testReadinessDuringRestarts() throws Exception { internalCluster().setBootstrapMasterNodeIndex(0); writeFileSettings(testJSON); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java index 22e7ce9e99edd..e68a60201931a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotsServiceIT.java @@ -10,7 +10,10 @@ import org.apache.logging.log4j.Level; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.SnapshotDeletionsInProgress; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; @@ -20,6 +23,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; @@ -111,6 +115,72 @@ public void testSnapshotDeletionFailureShouldBeLogged() throws Exception { } } + public void testDeleteSnapshotWhenNotWaitingForCompletion() throws Exception { + createIndexWithRandomDocs("test-index", randomIntBetween(1, 5)); + createRepository("test-repo", "mock"); + createSnapshot("test-repo", "test-snapshot", List.of("test-index")); + MockRepository repository = getRepositoryOnMaster("test-repo"); + PlainActionFuture listener = new PlainActionFuture<>(); + SubscribableListener snapshotDeletionListener = createSnapshotDeletionListener("test-repo"); + repository.blockOnDataFiles(); + try { + clusterAdmin().prepareDeleteSnapshot("test-repo", "test-snapshot").setWaitForCompletion(false).execute(listener); + // The request will complete as soon as the deletion is scheduled + safeGet(listener); + // The deletion won't complete until the block is removed + assertFalse(snapshotDeletionListener.isDone()); + } finally { + repository.unblock(); + } + safeAwait(snapshotDeletionListener); + } + + public void testDeleteSnapshotWhenWaitingForCompletion() throws Exception { + createIndexWithRandomDocs("test-index", randomIntBetween(1, 5)); + createRepository("test-repo", "mock"); + createSnapshot("test-repo", "test-snapshot", List.of("test-index")); + MockRepository repository = getRepositoryOnMaster("test-repo"); + PlainActionFuture requestCompleteListener = new PlainActionFuture<>(); + SubscribableListener snapshotDeletionListener = createSnapshotDeletionListener("test-repo"); + repository.blockOnDataFiles(); + try { + clusterAdmin().prepareDeleteSnapshot("test-repo", "test-snapshot").setWaitForCompletion(true).execute(requestCompleteListener); + // Neither the request nor the deletion will complete until we remove the block + assertFalse(requestCompleteListener.isDone()); + assertFalse(snapshotDeletionListener.isDone()); + } finally { + repository.unblock(); + } + safeGet(requestCompleteListener); + safeAwait(snapshotDeletionListener); + } + + /** + * Create a listener that completes once it has observed a snapshot delete begin and end for a specific repository + * + * @param repositoryName The repository to monitor for deletions + * @return the listener + */ + private SubscribableListener createSnapshotDeletionListener(String repositoryName) { + AtomicBoolean deleteHasStarted = new AtomicBoolean(false); + return ClusterServiceUtils.addTemporaryStateListener( + internalCluster().getCurrentMasterNodeInstance(ClusterService.class), + state -> { + SnapshotDeletionsInProgress deletionsInProgress = (SnapshotDeletionsInProgress) state.getCustoms() + .get(SnapshotDeletionsInProgress.TYPE); + if (deletionsInProgress == null) { + return false; + } + if (deleteHasStarted.get() == false) { + deleteHasStarted.set(deletionsInProgress.hasExecutingDeletion(repositoryName)); + return false; + } else { + return deletionsInProgress.hasExecutingDeletion(repositoryName) == false; + } + } + ); + } + public void testRerouteWhenShardSnapshotsCompleted() throws Exception { final var repoName = randomIdentifier(); createRepository(repoName, "mock"); diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index d31f25e4344b0..db7e3d40518ba 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -32,7 +32,7 @@ requires org.elasticsearch.plugin.analysis; requires org.elasticsearch.grok; requires org.elasticsearch.tdigest; - requires org.elasticsearch.vec; + requires org.elasticsearch.simdvec; requires com.sun.jna; requires hppc; @@ -431,6 +431,7 @@ org.elasticsearch.indices.IndicesFeatures, org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures, org.elasticsearch.index.mapper.MapperFeatures, + org.elasticsearch.script.ScriptFeatures, org.elasticsearch.search.retriever.RetrieversFeatures, org.elasticsearch.reservedstate.service.FileSettingsFeatures; diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 25e4b08d20115..ec02b8a45cd42 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -190,6 +190,12 @@ static TransportVersion def(int id) { public static final TransportVersion QUERY_RULE_CRUD_API_PUT = def(8_681_00_0); public static final TransportVersion DROP_UNUSED_NODES_REQUESTS = def(8_682_00_0); public static final TransportVersion QUERY_RULE_CRUD_API_GET_DELETE = def(8_683_00_0); + public static final TransportVersion MORE_LIGHTER_NODES_REQUESTS = def(8_684_00_0); + public static final TransportVersion DROP_UNUSED_NODES_IDS = def(8_685_00_0); + public static final TransportVersion DELETE_SNAPSHOTS_ASYNC_ADDED = def(8_686_00_0); + public static final TransportVersion VERSION_SUPPORTING_SPARSE_VECTOR_STATS = def(8_687_00_0); + public static final TransportVersion ML_AD_OUTPUT_MEMORY_ALLOCATOR_FIELD = def(8_688_00_0); + public static final TransportVersion FAILURE_STORE_LAZY_CREATION = def(8_689_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 06e4a1dd5368d..b2c78453d9c75 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -122,6 +122,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_7_17_20 = new Version(7_17_20_99); public static final Version V_7_17_21 = new Version(7_17_21_99); public static final Version V_7_17_22 = new Version(7_17_22_99); + public static final Version V_7_17_23 = new Version(7_17_23_99); public static final Version V_8_0_0 = new Version(8_00_00_99); public static final Version V_8_0_1 = new Version(8_00_01_99); @@ -176,6 +177,7 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_8_13_4 = new Version(8_13_04_99); public static final Version V_8_14_0 = new Version(8_14_00_99); public static final Version V_8_14_1 = new Version(8_14_01_99); + public static final Version V_8_14_2 = new Version(8_14_02_99); public static final Version V_8_15_0 = new Version(8_15_00_99); public static final Version CURRENT = V_8_15_0; diff --git a/server/src/main/java/org/elasticsearch/action/ActionListener.java b/server/src/main/java/org/elasticsearch/action/ActionListener.java index 21f3df2ab7175..ec01d88cb5e6e 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionListener.java +++ b/server/src/main/java/org/elasticsearch/action/ActionListener.java @@ -18,6 +18,7 @@ import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.RefCounted; import org.elasticsearch.core.Releasable; +import org.elasticsearch.transport.LeakTracker; import java.util.ArrayList; import java.util.List; @@ -425,6 +426,16 @@ public boolean equals(Object obj) { } } + /** + * @return A listener which (if assertions are enabled) wraps around the given delegate and asserts that it is called at least once. + */ + static ActionListener assertAtLeastOnce(ActionListener delegate) { + if (Assertions.ENABLED) { + return new ActionListenerImplementations.RunBeforeActionListener<>(delegate, LeakTracker.INSTANCE.track(delegate)::close); + } + return delegate; + } + /** * Execute the given action in a {@code try/catch} block which feeds all exceptions to the given listener's {@link #onFailure} method. */ diff --git a/server/src/main/java/org/elasticsearch/action/ActionModule.java b/server/src/main/java/org/elasticsearch/action/ActionModule.java index 5f56768138095..1c41f2cdff37d 100644 --- a/server/src/main/java/org/elasticsearch/action/ActionModule.java +++ b/server/src/main/java/org/elasticsearch/action/ActionModule.java @@ -248,6 +248,8 @@ import org.elasticsearch.plugins.ActionPlugin.ActionHandler; import org.elasticsearch.plugins.interceptor.RestServerActionPlugin; import org.elasticsearch.plugins.internal.RestExtension; +import org.elasticsearch.repositories.VerifyNodeRepositoryAction; +import org.elasticsearch.repositories.VerifyNodeRepositoryCoordinationAction; import org.elasticsearch.reservedstate.ReservedClusterStateHandler; import org.elasticsearch.reservedstate.service.ReservedClusterStateService; import org.elasticsearch.rest.RestController; @@ -649,6 +651,8 @@ public void reg actions.register(GetRepositoriesAction.INSTANCE, TransportGetRepositoriesAction.class); actions.register(TransportDeleteRepositoryAction.TYPE, TransportDeleteRepositoryAction.class); actions.register(VerifyRepositoryAction.INSTANCE, TransportVerifyRepositoryAction.class); + actions.register(VerifyNodeRepositoryCoordinationAction.TYPE, VerifyNodeRepositoryCoordinationAction.LocalAction.class); + actions.register(VerifyNodeRepositoryAction.TYPE, VerifyNodeRepositoryAction.TransportAction.class); actions.register(TransportCleanupRepositoryAction.TYPE, TransportCleanupRepositoryAction.class); actions.register(TransportGetSnapshotsAction.TYPE, TransportGetSnapshotsAction.class); actions.register(TransportDeleteSnapshotAction.TYPE, TransportDeleteSnapshotAction.class); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java index 054d6c7b1f6cc..467d2561f364e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequest.java @@ -8,118 +8,53 @@ package org.elasticsearch.action.admin.cluster.node.hotthreads; -import org.elasticsearch.TransportVersions; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.TimeValue; import org.elasticsearch.monitor.jvm.HotThreads; -import java.io.IOException; -import java.util.concurrent.TimeUnit; - public class NodesHotThreadsRequest extends BaseNodesRequest { - int threads = 3; - HotThreads.ReportType type = HotThreads.ReportType.CPU; - HotThreads.SortOrder sortOrder = HotThreads.SortOrder.TOTAL; - TimeValue interval = new TimeValue(500, TimeUnit.MILLISECONDS); - int snapshots = 10; - boolean ignoreIdleThreads = true; - - // for serialization - public NodesHotThreadsRequest(StreamInput in) throws IOException { - super(in); - threads = in.readInt(); - ignoreIdleThreads = in.readBoolean(); - type = HotThreads.ReportType.of(in.readString()); - interval = in.readTimeValue(); - snapshots = in.readInt(); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_7_16_0)) { - sortOrder = HotThreads.SortOrder.of(in.readString()); - } - } + final HotThreads.RequestOptions requestOptions; /** * Get hot threads from nodes based on the nodes ids specified. If none are passed, hot * threads for all nodes is used. */ - public NodesHotThreadsRequest(String... nodesIds) { + public NodesHotThreadsRequest(String[] nodesIds, HotThreads.RequestOptions requestOptions) { super(nodesIds); + this.requestOptions = requestOptions; } /** * Get hot threads from the given node, for use if the node isn't a stable member of the cluster. */ - public NodesHotThreadsRequest(DiscoveryNode node) { + public NodesHotThreadsRequest(DiscoveryNode node, HotThreads.RequestOptions requestOptions) { super(node); + this.requestOptions = requestOptions; } public int threads() { - return this.threads; - } - - public NodesHotThreadsRequest threads(int threads) { - this.threads = threads; - return this; + return requestOptions.threads(); } public boolean ignoreIdleThreads() { - return this.ignoreIdleThreads; - } - - public NodesHotThreadsRequest ignoreIdleThreads(boolean ignoreIdleThreads) { - this.ignoreIdleThreads = ignoreIdleThreads; - return this; - } - - public NodesHotThreadsRequest type(HotThreads.ReportType type) { - this.type = type; - return this; + return requestOptions.ignoreIdleThreads(); } public HotThreads.ReportType type() { - return this.type; - } - - public NodesHotThreadsRequest sortOrder(HotThreads.SortOrder sortOrder) { - this.sortOrder = sortOrder; - return this; + return requestOptions.reportType(); } public HotThreads.SortOrder sortOrder() { - return this.sortOrder; - } - - public NodesHotThreadsRequest interval(TimeValue interval) { - this.interval = interval; - return this; + return requestOptions.sortOrder(); } public TimeValue interval() { - return this.interval; + return requestOptions.interval(); } public int snapshots() { - return this.snapshots; - } - - public NodesHotThreadsRequest snapshots(int snapshots) { - this.snapshots = snapshots; - return this; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeInt(threads); - out.writeBoolean(ignoreIdleThreads); - out.writeString(type.getTypeValue()); - out.writeTimeValue(interval); - out.writeInt(snapshots); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_7_16_0)) { - out.writeString(sortOrder.getOrderValue()); - } + return requestOptions.snapshots(); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java index 89ef2efa4efa2..719a96ecb4d57 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/hotthreads/TransportNodesHotThreadsAction.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.admin.cluster.node.hotthreads; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -79,12 +80,12 @@ protected NodeHotThreads newNodeResponse(StreamInput in, DiscoveryNode node) thr @Override protected NodeHotThreads nodeOperation(NodeRequest request, Task task) { - final var hotThreads = new HotThreads().busiestThreads(request.request.threads) - .type(request.request.type) - .sortOrder(request.request.sortOrder) - .interval(request.request.interval) - .threadElementsSnapshotCount(request.request.snapshots) - .ignoreIdleThreads(request.request.ignoreIdleThreads); + final var hotThreads = new HotThreads().busiestThreads(request.requestOptions.threads()) + .type(request.requestOptions.reportType()) + .sortOrder(request.requestOptions.sortOrder()) + .interval(request.requestOptions.interval()) + .threadElementsSnapshotCount(request.requestOptions.snapshots()) + .ignoreIdleThreads(request.requestOptions.ignoreIdleThreads()); final var out = transportService.newNetworkBytesStream(); final var trackedResource = LeakTracker.wrap(out); var success = false; @@ -106,22 +107,23 @@ protected NodeHotThreads nodeOperation(NodeRequest request, Task task) { public static class NodeRequest extends TransportRequest { - // TODO don't wrap the whole top-level request, it contains heavy and irrelevant DiscoveryNode things; see #100878 - NodesHotThreadsRequest request; + final HotThreads.RequestOptions requestOptions; - public NodeRequest(StreamInput in) throws IOException { - super(in); - request = new NodesHotThreadsRequest(in); + NodeRequest(NodesHotThreadsRequest request) { + this.requestOptions = request.requestOptions; } - NodeRequest(NodesHotThreadsRequest request) { - this.request = request; + NodeRequest(StreamInput in) throws IOException { + super(in); + skipLegacyNodesRequestHeader(TransportVersions.MORE_LIGHTER_NODES_REQUESTS, in); + requestOptions = HotThreads.RequestOptions.readFrom(in); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - request.writeTo(out); + sendLegacyNodesRequestHeader(TransportVersions.MORE_LIGHTER_NODES_REQUESTS, out); + requestOptions.writeTo(out); } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java index 51699c1f7dcd3..cef9e880a8e70 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequest.java @@ -9,11 +9,7 @@ package org.elasticsearch.action.admin.cluster.node.info; import org.elasticsearch.action.support.nodes.BaseNodesRequest; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.UpdateForV9; -import java.io.IOException; import java.util.Set; import java.util.SortedSet; import java.util.TreeSet; @@ -25,18 +21,6 @@ public final class NodesInfoRequest extends BaseNodesRequest { private final NodesInfoMetrics nodesInfoMetrics; - /** - * Create a new NodeInfoRequest from a {@link StreamInput} object. - * - * @param in A stream input object. - * @throws IOException if the stream cannot be deserialized. - */ - @UpdateForV9 // this constructor is unused in v9 - public NodesInfoRequest(StreamInput in) throws IOException { - super(in); - nodesInfoMetrics = new NodesInfoMetrics(in); - } - /** * Get information from nodes based on the nodes ids specified. If none are passed, information * for all nodes will be returned. @@ -113,13 +97,6 @@ public NodesInfoRequest removeMetric(String metric) { return this; } - @UpdateForV9 // this method can just call localOnly() in v9 - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - nodesInfoMetrics.writeTo(out); - } - public NodesInfoMetrics getNodesInfoMetrics() { return nodesInfoMetrics; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java index ad6233717a334..4f5860b4ba50d 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/NodesInfoRequestBuilder.java @@ -14,8 +14,8 @@ // TODO: This class's interface should match that of NodesInfoRequest public class NodesInfoRequestBuilder extends NodesOperationRequestBuilder { - public NodesInfoRequestBuilder(ElasticsearchClient client) { - super(client, TransportNodesInfoAction.TYPE, new NodesInfoRequest()); + public NodesInfoRequestBuilder(ElasticsearchClient client, String[] nodeIds) { + super(client, TransportNodesInfoAction.TYPE, new NodesInfoRequest(nodeIds)); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java index 826d74935f556..ce962fb454a88 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/info/TransportNodesInfoAction.java @@ -101,11 +101,8 @@ public static class NodeInfoRequest extends TransportRequest { public NodeInfoRequest(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().onOrAfter(V_8_11_X)) { - this.nodesInfoMetrics = new NodesInfoMetrics(in); - } else { - this.nodesInfoMetrics = new NodesInfoRequest(in).getNodesInfoMetrics(); - } + skipLegacyNodesRequestHeader(V_8_11_X, in); + this.nodesInfoMetrics = new NodesInfoMetrics(in); } public NodeInfoRequest(NodesInfoRequest request) { @@ -115,11 +112,8 @@ public NodeInfoRequest(NodesInfoRequest request) { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getTransportVersion().onOrAfter(V_8_11_X)) { - this.nodesInfoMetrics.writeTo(out); - } else { - new NodesInfoRequest().clear().addMetrics(nodesInfoMetrics.requestedMetrics()).writeTo(out); - } + sendLegacyNodesRequestHeader(V_8_11_X, out); + nodesInfoMetrics.writeTo(out); } public Set requestedMetrics() { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java index a83a09af642fa..6894e68b49ed1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/NodesReloadSecureSettingsRequest.java @@ -9,7 +9,6 @@ package org.elasticsearch.action.admin.cluster.node.reload; import org.elasticsearch.TransportVersions; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.Strings; @@ -45,8 +44,8 @@ public class NodesReloadSecureSettingsRequest extends BaseNodesRequest Releasables.close(secureSettingsPassword))); - public NodesReloadSecureSettingsRequest() { - super((String[]) null); + public NodesReloadSecureSettingsRequest(String[] nodeIds) { + super(nodeIds); } public void setSecureStorePassword(SecureString secureStorePassword) { @@ -57,11 +56,6 @@ boolean hasPassword() { return this.secureSettingsPassword != null && this.secureSettingsPassword.length() > 0; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - @Override public void incRef() { refs.incRef(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java index 71da6fdeb1f3b..f906b7d659b7b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/reload/TransportNodesReloadSecureSettingsAction.java @@ -12,11 +12,11 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; @@ -88,20 +88,15 @@ protected NodesReloadSecureSettingsResponse.NodeResponse newNodeResponse(StreamI } @Override - protected void doExecute( - Task task, - NodesReloadSecureSettingsRequest request, - ActionListener listener - ) { - if (request.hasPassword() && isNodeLocal(request) == false && isNodeTransportTLSEnabled() == false) { - listener.onFailure( - new ElasticsearchException( - "Secure settings cannot be updated cluster wide when TLS for the transport layer" - + " is not enabled. Enable TLS or use the API with a `_local` filter on each node." - ) - ); + protected DiscoveryNode[] resolveRequest(NodesReloadSecureSettingsRequest request, ClusterState clusterState) { + final var concreteNodes = super.resolveRequest(request, clusterState); + final var isNodeLocal = concreteNodes.length == 1 && concreteNodes[0].getId().equals(clusterState.nodes().getLocalNodeId()); + if (request.hasPassword() && isNodeLocal == false && isNodeTransportTLSEnabled() == false) { + throw new ElasticsearchException(""" + Secure settings cannot be updated cluster wide when TLS for the transport layer is not enabled. Enable TLS or use the API \ + with a `_local` filter on each node."""); } else { - super.doExecute(task, request, listener); + return concreteNodes; } } @@ -148,13 +143,4 @@ protected NodesReloadSecureSettingsResponse.NodeResponse nodeOperation( private boolean isNodeTransportTLSEnabled() { return transportService.isTransportSecure(); } - - private boolean isNodeLocal(NodesReloadSecureSettingsRequest request) { - if (null == request.concreteNodes()) { - resolveRequest(request, clusterService.state()); - assert request.concreteNodes() != null; - } - final DiscoveryNode[] nodes = request.concreteNodes(); - return nodes.length == 1 && nodes[0].getId().equals(clusterService.localNode().getId()); - } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java index 7b636d766dbf2..b8070bef1e32b 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/shutdown/PrevalidateShardPathRequest.java @@ -8,12 +8,9 @@ package org.elasticsearch.action.admin.cluster.node.shutdown; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; -import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.shard.ShardId; -import java.io.IOException; import java.util.Arrays; import java.util.Objects; import java.util.Set; @@ -27,12 +24,6 @@ public PrevalidateShardPathRequest(Set shardIds, String... nodeIds) { this.shardIds = Set.copyOf(Objects.requireNonNull(shardIds)); } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - TransportAction.localOnly(); - } - public Set getShardIds() { return shardIds; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java index ff88bc5fcf464..c0329db1c1110 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequest.java @@ -11,14 +11,10 @@ import org.elasticsearch.action.admin.indices.stats.CommonStatsFlags; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import java.io.IOException; import java.util.Arrays; import java.util.Map; import java.util.Set; @@ -37,12 +33,6 @@ public NodesStatsRequest() { nodesStatsRequestParameters = new NodesStatsRequestParameters(); } - @UpdateForV9 // this constructor is unused in v9 - public NodesStatsRequest(StreamInput in) throws IOException { - super(in); - nodesStatsRequestParameters = new NodesStatsRequestParameters(in); - } - /** * Get stats from nodes based on the nodes ids specified. If none are passed, stats * for all nodes will be returned. @@ -179,13 +169,6 @@ public void setIncludeShardsStats(boolean includeShardsStats) { nodesStatsRequestParameters.setIncludeShardsStats(includeShardsStats); } - @UpdateForV9 // this method can just call localOnly() in v9 - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - nodesStatsRequestParameters.writeTo(out); - } - public NodesStatsRequestParameters getNodesStatsRequestParameters() { return nodesStatsRequestParameters; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java index 8d863653874bb..b412f738f5e4c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestBuilder.java @@ -17,8 +17,8 @@ public class NodesStatsRequestBuilder extends NodesOperationRequestBuilder< NodesStatsResponse, NodesStatsRequestBuilder> { - public NodesStatsRequestBuilder(ElasticsearchClient client) { - super(client, TransportNodesStatsAction.TYPE, new NodesStatsRequest()); + public NodesStatsRequestBuilder(ElasticsearchClient client, String[] nodeIds) { + super(client, TransportNodesStatsAction.TYPE, new NodesStatsRequest(nodeIds)); } /** diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java index a84905e7f4c8e..1b7ce13333891 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java @@ -158,23 +158,19 @@ protected NodeStats nodeOperation(NodeStatsRequest request, Task task) { public static class NodeStatsRequest extends TransportRequest { private final NodesStatsRequestParameters nodesStatsRequestParameters; - private final String[] nodesIds; public NodeStatsRequest(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { - this.nodesStatsRequestParameters = new NodesStatsRequestParameters(in); - this.nodesIds = in.readStringArray(); - } else { - final NodesStatsRequest nodesStatsRequest = new NodesStatsRequest(in); - this.nodesStatsRequestParameters = nodesStatsRequest.getNodesStatsRequestParameters(); - this.nodesIds = nodesStatsRequest.nodesIds(); + skipLegacyNodesRequestHeader(TransportVersions.V_8_13_0, in); + this.nodesStatsRequestParameters = new NodesStatsRequestParameters(in); + if (in.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0) + && in.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_IDS)) { + in.readStringArray(); // formerly nodeIds, now unused } } NodeStatsRequest(NodesStatsRequest request) { this.nodesStatsRequestParameters = request.getNodesStatsRequestParameters(); - this.nodesIds = request.nodesIds(); } @Override @@ -183,8 +179,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, @Override public String getDescription() { return Strings.format( - "nodes=%s, metrics=%s, flags=%s", - Arrays.toString(nodesIds), + "metrics=%s, flags=%s", nodesStatsRequestParameters.requestedMetrics().toString(), Arrays.toString(nodesStatsRequestParameters.indices().getFlags()) ); @@ -195,11 +190,11 @@ public String getDescription() { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { - this.nodesStatsRequestParameters.writeTo(out); - out.writeStringArrayNullable(nodesIds); - } else { - new NodesStatsRequest(nodesStatsRequestParameters, this.nodesIds).writeTo(out); + sendLegacyNodesRequestHeader(TransportVersions.V_8_13_0, out); + nodesStatsRequestParameters.writeTo(out); + if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0) + && out.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_IDS)) { + out.writeStringArray(Strings.EMPTY_ARRAY); // formerly nodeIds, now unused } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java index 6789c7ae81441..31fd93685e2c6 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/NodesUsageRequest.java @@ -9,22 +9,12 @@ package org.elasticsearch.action.admin.cluster.node.usage; import org.elasticsearch.action.support.nodes.BaseNodesRequest; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; public class NodesUsageRequest extends BaseNodesRequest { private boolean restActions; private boolean aggregations; - public NodesUsageRequest(StreamInput in) throws IOException { - super(in); - this.restActions = in.readBoolean(); - this.aggregations = in.readBoolean(); - } - /** * Get usage from nodes based on the nodes ids specified. If none are * passed, usage for all nodes will be returned. @@ -79,11 +69,4 @@ public NodesUsageRequest aggregations(boolean aggregations) { this.aggregations = aggregations; return this; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeBoolean(restActions); - out.writeBoolean(aggregations); - } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java index 638773cce52e8..72bbe2683d157 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/node/usage/TransportNodesUsageAction.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.node.usage; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; @@ -77,31 +78,35 @@ protected NodeUsage newNodeResponse(StreamInput in, DiscoveryNode node) throws I } @Override - protected NodeUsage nodeOperation(NodeUsageRequest nodeUsageRequest, Task task) { - NodesUsageRequest request = nodeUsageRequest.request; - Map restUsage = request.restActions() ? restUsageService.getRestUsageStats() : null; - Map aggsUsage = request.aggregations() ? aggregationUsageService.getUsageStats() : null; + protected NodeUsage nodeOperation(NodeUsageRequest request, Task task) { + Map restUsage = request.restActions ? restUsageService.getRestUsageStats() : null; + Map aggsUsage = request.aggregations ? aggregationUsageService.getUsageStats() : null; return new NodeUsage(clusterService.localNode(), System.currentTimeMillis(), sinceTime, restUsage, aggsUsage); } public static class NodeUsageRequest extends TransportRequest { - // TODO don't wrap the whole top-level request, it contains heavy and irrelevant DiscoveryNode things; see #100878 - NodesUsageRequest request; + final boolean restActions; + final boolean aggregations; public NodeUsageRequest(StreamInput in) throws IOException { super(in); - request = new NodesUsageRequest(in); + skipLegacyNodesRequestHeader(TransportVersions.MORE_LIGHTER_NODES_REQUESTS, in); + restActions = in.readBoolean(); + aggregations = in.readBoolean(); } NodeUsageRequest(NodesUsageRequest request) { - this.request = request; + restActions = request.restActions(); + aggregations = request.aggregations(); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - request.writeTo(out); + sendLegacyNodesRequestHeader(TransportVersions.MORE_LIGHTER_NODES_REQUESTS, out); + out.writeBoolean(restActions); + out.writeBoolean(aggregations); } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java index 67389ea3116d8..2356087d64e41 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/delete/DeleteSnapshotRequest.java @@ -8,6 +8,7 @@ package org.elasticsearch.action.admin.cluster.snapshots.delete; +import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.common.Strings; @@ -30,6 +31,7 @@ public class DeleteSnapshotRequest extends MasterNodeRequest listener) { + if (clusterService.state().getMinTransportVersion().before(TransportVersions.DELETE_SNAPSHOTS_ASYNC_ADDED) + && request.waitForCompletion() == false) { + throw new UnsupportedOperationException("wait_for_completion parameter is not supported by all nodes in this cluster"); + } + super.doExecute(task, request, listener); + } + @Override protected void masterOperation( Task task, diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java index 202940dfe7f69..82f32d2472b97 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/status/TransportNodesSnapshotsStatus.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.action.support.nodes.BaseNodesResponse; @@ -132,11 +131,6 @@ public Request snapshots(Snapshot[] snapshots) { this.snapshots = snapshots; return this; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } } public static class NodesSnapshotStatus extends BaseNodesResponse { diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java index 20626f31e8006..8e3b41a4876d4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsIndices.java @@ -14,6 +14,7 @@ import org.elasticsearch.index.fielddata.FieldDataStats; import org.elasticsearch.index.shard.DenseVectorStats; import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.xcontent.ToXContentFragment; @@ -38,6 +39,7 @@ public class ClusterStatsIndices implements ToXContentFragment { private final MappingStats mappings; private final VersionStats versions; private final DenseVectorStats denseVectorStats; + private final SparseVectorStats sparseVectorStats; public ClusterStatsIndices( List nodeResponses, @@ -55,6 +57,7 @@ public ClusterStatsIndices( this.completion = new CompletionStats(); this.segments = new SegmentsStats(); this.denseVectorStats = new DenseVectorStats(); + this.sparseVectorStats = new SparseVectorStats(); for (ClusterStatsNodeResponse r : nodeResponses) { for (org.elasticsearch.action.admin.indices.stats.ShardStats shardStats : r.shardsStats()) { @@ -71,13 +74,14 @@ public ClusterStatsIndices( if (shardStats.getShardRouting().primary()) { indexShardStats.primaries++; docs.add(shardCommonStats.getDocs()); + denseVectorStats.add(shardCommonStats.getDenseVectorStats()); + sparseVectorStats.add(shardCommonStats.getSparseVectorStats()); } store.add(shardCommonStats.getStore()); fieldData.add(shardCommonStats.getFieldData()); queryCache.add(shardCommonStats.getQueryCache()); completion.add(shardCommonStats.getCompletion()); segments.add(shardCommonStats.getSegments()); - denseVectorStats.add(shardCommonStats.getDenseVectorStats()); } searchUsageStats.add(r.searchUsageStats()); @@ -146,6 +150,10 @@ public DenseVectorStats getDenseVectorStats() { return denseVectorStats; } + public SparseVectorStats getSparseVectorStats() { + return sparseVectorStats; + } + static final class Fields { static final String COUNT = "count"; } @@ -171,6 +179,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } searchUsageStats.toXContent(builder, params); denseVectorStats.toXContent(builder, params); + sparseVectorStats.toXContent(builder, params); return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java index bba669e07a70c..77652eeb7d94e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsRequest.java @@ -9,26 +9,16 @@ package org.elasticsearch.action.admin.cluster.stats; import org.elasticsearch.action.support.nodes.BaseNodesRequest; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.UpdateForV9; import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; -import java.io.IOException; import java.util.Map; /** * A request to get cluster level stats. */ public class ClusterStatsRequest extends BaseNodesRequest { - - @UpdateForV9 // this constructor is unused in v9 - public ClusterStatsRequest(StreamInput in) throws IOException { - super(in); - } - /** * Get stats from nodes based on the nodes ids specified. If none are passed, stats * based on all nodes will be returned. @@ -41,11 +31,4 @@ public ClusterStatsRequest(String... nodesIds) { public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { return new CancellableTask(id, type, action, "", parentTaskId, headers); } - - @UpdateForV9 // this method can just call localOnly() in v9 - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java index e2ade5060c476..afd13b02ab3f2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/MappingStats.java @@ -230,7 +230,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVLong(totalFieldCount); out.writeOptionalVLong(totalDeduplicatedFieldCount); out.writeOptionalVLong(totalMappingSizeBytes); - } // else just omit these stats, they're not computed on older nodes anyway + } out.writeCollection(fieldTypeStats); out.writeCollection(runtimeFieldStats); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java index 2a8fecde7ee9e..5d12cb5c0f657 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/TransportClusterStatsAction.java @@ -72,7 +72,8 @@ public class TransportClusterStatsAction extends TransportNodesAction< CommonStatsFlags.Flag.QueryCache, CommonStatsFlags.Flag.Completion, CommonStatsFlags.Flag.Segments, - CommonStatsFlags.Flag.DenseVector + CommonStatsFlags.Flag.DenseVector, + CommonStatsFlags.Flag.SparseVector ); private final NodeService nodeService; @@ -260,9 +261,7 @@ public static class ClusterStatsNodeRequest extends TransportRequest { public ClusterStatsNodeRequest(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_REQUESTS)) { - new ClusterStatsRequest(in); - } + skipLegacyNodesRequestHeader(TransportVersions.DROP_UNUSED_NODES_REQUESTS, in); } @Override @@ -273,9 +272,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_REQUESTS)) { - new ClusterStatsRequest().writeTo(out); - } + sendLegacyNodesRequestHeader(TransportVersions.DROP_UNUSED_NODES_REQUESTS, out); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index 094fccbc35182..e68263aab5330 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -261,7 +261,8 @@ ClusterState execute( ClusterState clusterState = metadataCreateDataStreamService.createDataStream( createRequest, currentState, - rerouteCompletionIsNotRequired() + rerouteCompletionIsNotRequired(), + request.isInitializeFailureStore() ); final var dataStream = clusterState.metadata().dataStreams().get(request.index()); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java index 3a78738ae986a..5d1b7264ebf81 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java @@ -64,6 +64,8 @@ public class CreateIndexRequest extends AcknowledgedRequest private boolean requireDataStream; + private boolean initializeFailureStore; + private Settings settings = Settings.EMPTY; private String mappings = "{}"; @@ -109,6 +111,11 @@ public CreateIndexRequest(StreamInput in) throws IOException { } else { requireDataStream = false; } + if (in.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_LAZY_CREATION)) { + initializeFailureStore = in.readBoolean(); + } else { + initializeFailureStore = true; + } } public CreateIndexRequest() { @@ -468,6 +475,19 @@ public CreateIndexRequest requireDataStream(boolean requireDataStream) { return this; } + public boolean isInitializeFailureStore() { + return initializeFailureStore; + } + + /** + * Set whether this CreateIndexRequest should initialize the failure store on data stream creation. This can be necessary when, for + * example, a failure occurs while trying to ingest a document into a data stream that has to be auto-created. + */ + public CreateIndexRequest initializeFailureStore(boolean initializeFailureStore) { + this.initializeFailureStore = initializeFailureStore; + return this; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -491,7 +511,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(origin); } if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_13_0)) { - out.writeOptionalBoolean(this.requireDataStream); + out.writeBoolean(this.requireDataStream); + } + if (out.getTransportVersion().onOrAfter(TransportVersions.FAILURE_STORE_LAZY_CREATION)) { + out.writeBoolean(this.initializeFailureStore); } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java index e0f699121de8c..1f6d6e65b9128 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/find/FindDanglingIndexRequest.java @@ -8,12 +8,8 @@ package org.elasticsearch.action.admin.indices.dangling.find; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; public class FindDanglingIndexRequest extends BaseNodesRequest { private final String indexUUID; @@ -31,9 +27,4 @@ public String getIndexUUID() { public String toString() { return "FindDanglingIndicesRequest{indexUUID='" + indexUUID + "'}"; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java index f0eafe3d0bf8c..450c45bf0742c 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/dangling/list/ListDanglingIndicesRequest.java @@ -8,12 +8,8 @@ package org.elasticsearch.action.admin.indices.dangling.list; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.stream.StreamOutput; - -import java.io.IOException; public class ListDanglingIndicesRequest extends BaseNodesRequest { /** @@ -39,9 +35,4 @@ public String getIndexUUID() { public String toString() { return "ListDanglingIndicesRequest{indexUUID='" + indexUUID + "'}"; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java index ed3721b35f3b4..5c7518abdbbf8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java @@ -84,6 +84,7 @@ public class MetadataRolloverService { AutoShardingType.COOLDOWN_PREVENTED_DECREASE, "es.auto_sharding.cooldown_prevented_decrease.total" ); + private static final String NON_EXISTENT_SOURCE = "_none_"; private final ThreadPool threadPool; private final MetadataCreateIndexService createIndexService; @@ -221,14 +222,13 @@ private static NameResolution resolveAliasRolloverNames(Metadata metadata, Index private static NameResolution resolveDataStreamRolloverNames(Metadata metadata, DataStream dataStream, boolean isFailureStoreRollover) { final DataStream.DataStreamIndices dataStreamIndices = dataStream.getDataStreamIndices(isFailureStoreRollover); - assert dataStreamIndices.getWriteIndex() != null : "Unable to roll over dataStreamIndices with no indices"; + assert dataStreamIndices.getIndices().isEmpty() == false || isFailureStoreRollover + : "Unable to roll over dataStreamIndices with no indices"; - final IndexMetadata originalWriteIndex = metadata.index(dataStreamIndices.getWriteIndex()); - return new NameResolution( - originalWriteIndex.getIndex().getName(), - null, - dataStream.nextWriteIndexAndGeneration(metadata, dataStreamIndices).v1() - ); + final String originalWriteIndex = dataStreamIndices.getIndices().isEmpty() && dataStreamIndices.isRolloverOnWrite() + ? NON_EXISTENT_SOURCE + : metadata.index(dataStreamIndices.getWriteIndex()).getIndex().getName(); + return new NameResolution(originalWriteIndex, null, dataStream.nextWriteIndexAndGeneration(metadata, dataStreamIndices).v1()); } private RolloverResult rolloverAlias( @@ -323,13 +323,14 @@ private RolloverResult rolloverDataStream( } final DataStream.DataStreamIndices dataStreamIndices = dataStream.getDataStreamIndices(isFailureStoreRollover); - final Index originalWriteIndex = dataStreamIndices.getWriteIndex(); + final boolean isLazyCreation = dataStreamIndices.getIndices().isEmpty() && dataStreamIndices.isRolloverOnWrite(); + final Index originalWriteIndex = isLazyCreation ? null : dataStreamIndices.getWriteIndex(); final Tuple nextIndexAndGeneration = dataStream.nextWriteIndexAndGeneration(metadata, dataStreamIndices); final String newWriteIndexName = nextIndexAndGeneration.v1(); final long newGeneration = nextIndexAndGeneration.v2(); MetadataCreateIndexService.validateIndexName(newWriteIndexName, currentState); // fails if the index already exists if (onlyValidate) { - return new RolloverResult(newWriteIndexName, originalWriteIndex.getName(), currentState); + return new RolloverResult(newWriteIndexName, isLazyCreation ? NON_EXISTENT_SOURCE : originalWriteIndex.getName(), currentState); } ClusterState newState; @@ -423,10 +424,12 @@ yield new DataStreamAutoShardingEvent( RolloverInfo rolloverInfo = new RolloverInfo(dataStreamName, metConditions, threadPool.absoluteTimeInMillis()); - Metadata.Builder metadataBuilder = Metadata.builder(newState.metadata()) - .put( + Metadata.Builder metadataBuilder = Metadata.builder(newState.metadata()); + if (isLazyCreation == false) { + metadataBuilder.put( IndexMetadata.builder(newState.metadata().index(originalWriteIndex)).stats(sourceIndexStats).putRolloverInfo(rolloverInfo) ); + } metadataBuilder = writeLoadForecaster.withWriteLoadForecastForWriteIndex(dataStreamName, metadataBuilder); metadataBuilder = withShardSizeForecastForWriteIndex(dataStreamName, metadataBuilder); @@ -434,7 +437,7 @@ yield new DataStreamAutoShardingEvent( newState = ClusterState.builder(newState).metadata(metadataBuilder).build(); newState = MetadataDataStreamsService.setRolloverOnWrite(newState, dataStreamName, false, isFailureStoreRollover); - return new RolloverResult(newWriteIndexName, originalWriteIndex.getName(), newState); + return new RolloverResult(newWriteIndexName, isLazyCreation ? NON_EXISTENT_SOURCE : originalWriteIndex.getName(), newState); } /** @@ -664,12 +667,6 @@ static void validate( "aliases, mappings, and index settings may not be specified when rolling over a data stream" ); } - var dataStream = (DataStream) indexAbstraction; - if (isFailureStoreRollover && dataStream.isFailureStoreEnabled() == false) { - throw new IllegalArgumentException( - "unable to roll over failure store because [" + indexAbstraction.getName() + "] does not have the failure store enabled" - ); - } } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index bf059f6fe868e..34da6795cd5f2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -230,6 +230,16 @@ protected void masterOperation( ); return; } + if (targetFailureStore && rolloverTargetAbstraction.isDataStreamRelated() == false) { + listener.onFailure(new IllegalStateException("Rolling over failure stores is only possible on data streams.")); + return; + } + + // When we're initializing a failure store, we skip the stats request because there is no source index to retrieve stats for. + if (targetFailureStore && ((DataStream) rolloverTargetAbstraction).getFailureIndices().getIndices().isEmpty()) { + initializeFailureStore(rolloverRequest, listener, trialSourceIndexName, trialRolloverIndexName); + return; + } final var statsIndicesOptions = new IndicesOptions( IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS, @@ -317,7 +327,7 @@ protected void masterOperation( // Pre-check the conditions to see whether we should submit a new cluster state task if (rolloverRequest.areConditionsMet(trialConditionResults)) { - String source = "rollover_index source [" + trialRolloverIndexName + "] to target [" + trialRolloverIndexName + "]"; + String source = "rollover_index source [" + trialSourceIndexName + "] to target [" + trialRolloverIndexName + "]"; RolloverTask rolloverTask = new RolloverTask( rolloverRequest, statsResponse, @@ -334,6 +344,40 @@ protected void masterOperation( ); } + private void initializeFailureStore( + RolloverRequest rolloverRequest, + ActionListener listener, + String trialSourceIndexName, + String trialRolloverIndexName + ) { + if (rolloverRequest.getConditionValues().isEmpty() == false) { + listener.onFailure( + new IllegalStateException("Rolling over/initializing an empty failure store is only supported without conditions.") + ); + return; + } + final RolloverResponse trialRolloverResponse = new RolloverResponse( + trialSourceIndexName, + trialRolloverIndexName, + Map.of(), + rolloverRequest.isDryRun(), + false, + false, + false, + rolloverRequest.isLazy() + ); + + // If this is a dry run, return with the results without invoking a cluster state update. + if (rolloverRequest.isDryRun()) { + listener.onResponse(trialRolloverResponse); + return; + } + + String source = "initialize_failure_store with index [" + trialRolloverIndexName + "]"; + RolloverTask rolloverTask = new RolloverTask(rolloverRequest, null, trialRolloverResponse, null, listener); + submitRolloverTask(rolloverRequest, source, rolloverTask); + } + void submitRolloverTask(RolloverRequest rolloverRequest, String source, RolloverTask rolloverTask) { rolloverTaskQueue.submitTask(source, rolloverTask, rolloverRequest.masterNodeTimeout()); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java index b6345ed0fce4a..2596e62c85259 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.index.shard.ShardCountStats; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.warmer.WarmerStats; @@ -45,6 +46,8 @@ import java.io.IOException; import java.util.Objects; +import static org.elasticsearch.TransportVersions.VERSION_SUPPORTING_SPARSE_VECTOR_STATS; + public class CommonStats implements Writeable, ToXContentFragment { private static final TransportVersion VERSION_SUPPORTING_NODE_MAPPINGS = TransportVersions.V_8_5_0; @@ -110,6 +113,9 @@ public class CommonStats implements Writeable, ToXContentFragment { @Nullable public DenseVectorStats denseVectorStats; + @Nullable + public SparseVectorStats sparseVectorStats; + public CommonStats() { this(CommonStatsFlags.NONE); } @@ -139,6 +145,7 @@ public CommonStats(CommonStatsFlags flags) { case Shards -> shards = new ShardCountStats(); case Mappings -> nodeMappings = new NodeMappingStats(); case DenseVector -> denseVectorStats = new DenseVectorStats(); + case SparseVector -> sparseVectorStats = new SparseVectorStats(); default -> throw new IllegalStateException("Unknown Flag: " + flag); } } @@ -182,6 +189,7 @@ public static CommonStats getShardLevelStats(IndicesQueryCache indicesQueryCache // Setting to 1 because the single IndexShard passed to this method implies 1 shard stats.shards = new ShardCountStats(1); case DenseVector -> stats.denseVectorStats = indexShard.denseVectorStats(); + case SparseVector -> stats.sparseVectorStats = indexShard.sparseVectorStats(); default -> throw new IllegalStateException("Unknown or invalid flag for shard-level stats: " + flag); } } catch (AlreadyClosedException e) { @@ -219,6 +227,9 @@ public CommonStats(StreamInput in) throws IOException { if (in.getTransportVersion().onOrAfter(VERSION_SUPPORTING_DENSE_VECTOR_STATS)) { denseVectorStats = in.readOptionalWriteable(DenseVectorStats::new); } + if (in.getTransportVersion().onOrAfter(VERSION_SUPPORTING_SPARSE_VECTOR_STATS)) { + sparseVectorStats = in.readOptionalWriteable(SparseVectorStats::new); + } } @Override @@ -249,6 +260,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(VERSION_SUPPORTING_DENSE_VECTOR_STATS)) { out.writeOptionalWriteable(denseVectorStats); } + if (out.getTransportVersion().onOrAfter(VERSION_SUPPORTING_SPARSE_VECTOR_STATS)) { + out.writeOptionalWriteable(sparseVectorStats); + } } @Override @@ -275,7 +289,8 @@ public boolean equals(Object o) { && Objects.equals(bulk, that.bulk) && Objects.equals(shards, that.shards) && Objects.equals(nodeMappings, that.nodeMappings) - && Objects.equals(denseVectorStats, that.denseVectorStats); + && Objects.equals(denseVectorStats, that.denseVectorStats) + && Objects.equals(sparseVectorStats, that.sparseVectorStats); } @Override @@ -300,7 +315,8 @@ public int hashCode() { bulk, shards, nodeMappings, - denseVectorStats + denseVectorStats, + sparseVectorStats ); } @@ -465,6 +481,14 @@ public void add(CommonStats stats) { } else { denseVectorStats.add(stats.getDenseVectorStats()); } + if (sparseVectorStats == null) { + if (stats.getSparseVectorStats() != null) { + sparseVectorStats = new SparseVectorStats(); + sparseVectorStats.add(stats.getSparseVectorStats()); + } + } else { + sparseVectorStats.add(stats.getSparseVectorStats()); + } } @Nullable @@ -567,6 +591,11 @@ public DenseVectorStats getDenseVectorStats() { return denseVectorStats; } + @Nullable + public SparseVectorStats getSparseVectorStats() { + return sparseVectorStats; + } + /** * Utility method which computes total memory by adding * FieldData, PercolatorCache, Segments (index writer, version map) @@ -609,6 +638,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws addIfNonNull(builder, params, bulk); addIfNonNull(builder, params, nodeMappings); addIfNonNull(builder, params, denseVectorStats); + addIfNonNull(builder, params, sparseVectorStats); return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java index 391ac532a0c3a..31dc6a744a757 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java @@ -223,7 +223,8 @@ public enum Flag { Bulk("bulk", 17), Shards("shard_stats", 18), Mappings("mappings", 19), - DenseVector("dense_vector", 20); + DenseVector("dense_vector", 20), + SparseVector("sparse_vector", 21); private final String restName; private final int index; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java index 7cd32811e3638..12c26f3b10ca4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java @@ -279,6 +279,15 @@ public boolean denseVector() { return flags.isSet(Flag.DenseVector); } + public IndicesStatsRequest sparseVector(boolean sparseVector) { + flags.set(Flag.SparseVector, sparseVector); + return this; + } + + public boolean sparseVector() { + return flags.isSet(Flag.SparseVector); + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java index 40d0c0998b4e7..0c20869eeb906 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java @@ -159,4 +159,9 @@ public IndicesStatsRequestBuilder setDenseVector(boolean denseVector) { request.denseVector(denseVector); return this; } + + public IndicesStatsRequestBuilder setSparseVector(boolean sparseVector) { + request.sparseVector(sparseVector); + return this; + } } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java index 5dccd1b55f554..75ab08de942dc 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkRequestParser.java @@ -341,58 +341,25 @@ public void parse( // we use internalAdd so we don't fork here, this allows us not to copy over the big byte array to small chunks // of index request. - if ("index".equals(action)) { - if (opType == null) { - indexRequestConsumer.accept( - new IndexRequest(index).id(id) - .routing(routing) - .version(version) - .versionType(versionType) - .setPipeline(pipeline) - .setIfSeqNo(ifSeqNo) - .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) - .setDynamicTemplates(dynamicTemplates) - .setRequireAlias(requireAlias) - .setRequireDataStream(requireDataStream) - .setListExecutedPipelines(listExecutedPipelines), - type - ); - } else { - indexRequestConsumer.accept( - new IndexRequest(index).id(id) - .routing(routing) - .version(version) - .versionType(versionType) - .create("create".equals(opType)) - .setPipeline(pipeline) - .setIfSeqNo(ifSeqNo) - .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) - .setDynamicTemplates(dynamicTemplates) - .setRequireAlias(requireAlias) - .setRequireDataStream(requireDataStream) - .setListExecutedPipelines(listExecutedPipelines), - type - ); + if ("index".equals(action) || "create".equals(action)) { + var indexRequest = new IndexRequest(index).id(id) + .routing(routing) + .version(version) + .versionType(versionType) + .setPipeline(pipeline) + .setIfSeqNo(ifSeqNo) + .setIfPrimaryTerm(ifPrimaryTerm) + .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) + .setDynamicTemplates(dynamicTemplates) + .setRequireAlias(requireAlias) + .setRequireDataStream(requireDataStream) + .setListExecutedPipelines(listExecutedPipelines); + if ("create".equals(action)) { + indexRequest = indexRequest.create(true); + } else if (opType != null) { + indexRequest = indexRequest.create("create".equals(opType)); } - } else if ("create".equals(action)) { - indexRequestConsumer.accept( - new IndexRequest(index).id(id) - .routing(routing) - .version(version) - .versionType(versionType) - .create(true) - .setPipeline(pipeline) - .setIfSeqNo(ifSeqNo) - .setIfPrimaryTerm(ifPrimaryTerm) - .source(sliceTrimmingCarriageReturn(data, from, nextMarker, xContentType), xContentType) - .setDynamicTemplates(dynamicTemplates) - .setRequireAlias(requireAlias) - .setRequireDataStream(requireDataStream) - .setListExecutedPipelines(listExecutedPipelines), - type - ); + indexRequestConsumer.accept(indexRequest, type); } else if ("update".equals(action)) { if (version != Versions.MATCH_ANY || versionType != VersionType.INTERNAL) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index a9431ca1eeff0..4fc17407ae6d0 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -350,7 +350,7 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, Executor ex return; } - Map indicesToAutoCreate = new HashMap<>(); + Map indicesToAutoCreate = new HashMap<>(); Set dataStreamsToBeRolledOver = new HashSet<>(); Set failureStoresToBeRolledOver = new HashSet<>(); populateMissingTargets(bulkRequest, indicesToAutoCreate, dataStreamsToBeRolledOver, failureStoresToBeRolledOver); @@ -373,19 +373,19 @@ protected void doInternalExecute(Task task, BulkRequest bulkRequest, Executor ex * for lazy rollover. * * @param bulkRequest the bulk request - * @param indicesToAutoCreate a map of index names to whether they require a data stream + * @param indicesToAutoCreate a map of index names to their creation request that need to be auto-created * @param dataStreamsToBeRolledOver a set of data stream names that were marked for lazy rollover and thus need to be rolled over now * @param failureStoresToBeRolledOver a set of data stream names whose failure store was marked for lazy rollover and thus need to be * rolled over now */ private void populateMissingTargets( BulkRequest bulkRequest, - Map indicesToAutoCreate, + Map indicesToAutoCreate, Set dataStreamsToBeRolledOver, Set failureStoresToBeRolledOver ) { ClusterState state = clusterService.state(); - // A map for memorizing which indices we already exist (or don't). + // A map for memorizing which indices exist. Map indexExistence = new HashMap<>(); Function indexExistenceComputation = (index) -> indexNameExpressionResolver.hasIndexAbstraction(index, state); boolean lazyRolloverFeature = featureService.clusterHasFeature(state, LazyRolloverAction.DATA_STREAM_LAZY_ROLLOVER); @@ -399,19 +399,36 @@ private void populateMissingTargets( && request.versionType() != VersionType.EXTERNAL_GTE) { continue; } + boolean writeToFailureStore = request instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore(); boolean indexExists = indexExistence.computeIfAbsent(request.index(), indexExistenceComputation); if (indexExists == false) { - // We should only auto create an index if _none_ of the requests are requiring it to be an alias. + // We should only auto-create an index if _none_ of the requests are requiring it to be an alias. if (request.isRequireAlias()) { - // Remember that this a request required this index to be an alias. + // Remember that this request required this index to be an alias. if (indicesThatRequireAlias.add(request.index())) { // If we didn't already know that, we remove the index from the list of indices to create (if present). indicesToAutoCreate.remove(request.index()); } } else if (indicesThatRequireAlias.contains(request.index()) == false) { - Boolean requiresDataStream = indicesToAutoCreate.get(request.index()); - if (requiresDataStream == null || (requiresDataStream == false && request.isRequireDataStream())) { - indicesToAutoCreate.put(request.index(), request.isRequireDataStream()); + CreateIndexRequest createIndexRequest = indicesToAutoCreate.get(request.index()); + // Create a new CreateIndexRequest if we didn't already have one. + if (createIndexRequest == null) { + createIndexRequest = new CreateIndexRequest(request.index()).cause("auto(bulk api)") + .masterNodeTimeout(bulkRequest.timeout()) + .requireDataStream(request.isRequireDataStream()) + // If this IndexRequest is directed towards a failure store, but the data stream doesn't exist, we initialize + // the failure store on data stream creation instead of lazily. + .initializeFailureStore(writeToFailureStore); + indicesToAutoCreate.put(request.index(), createIndexRequest); + } else { + // Track whether one of the index requests in this bulk request requires the target to be a data stream. + if (createIndexRequest.isRequireDataStream() == false && request.isRequireDataStream()) { + createIndexRequest.requireDataStream(true); + } + // Track whether one of the index requests in this bulk request is directed towards a failure store. + if (createIndexRequest.isInitializeFailureStore() == false && writeToFailureStore) { + createIndexRequest.initializeFailureStore(true); + } } } } @@ -419,7 +436,6 @@ private void populateMissingTargets( if (lazyRolloverFeature) { DataStream dataStream = state.metadata().dataStreams().get(request.index()); if (dataStream != null) { - var writeToFailureStore = request instanceof IndexRequest indexRequest && indexRequest.isWriteToFailureStore(); if (writeToFailureStore == false && dataStream.getBackingIndices().isRolloverOnWrite()) { dataStreamsToBeRolledOver.add(request.index()); } else if (lazyRolloverFailureStoreFeature @@ -441,7 +457,7 @@ protected void createMissingIndicesAndIndexData( BulkRequest bulkRequest, Executor executor, ActionListener listener, - Map indicesToAutoCreate, + Map indicesToAutoCreate, Set dataStreamsToBeRolledOver, Set failureStoresToBeRolledOver, long startTime @@ -468,14 +484,14 @@ protected void doRun() { private void createIndices( BulkRequest bulkRequest, - Map indicesToAutoCreate, + Map indicesToAutoCreate, Map indicesThatCannotBeCreated, AtomicArray responses, RefCountingRunnable refs ) { - for (Map.Entry indexEntry : indicesToAutoCreate.entrySet()) { + for (Map.Entry indexEntry : indicesToAutoCreate.entrySet()) { final String index = indexEntry.getKey(); - createIndex(index, indexEntry.getValue(), bulkRequest.timeout(), ActionListener.releaseAfter(new ActionListener<>() { + createIndex(indexEntry.getValue(), ActionListener.releaseAfter(new ActionListener<>() { @Override public void onResponse(CreateIndexResponse createIndexResponse) {} @@ -641,12 +657,7 @@ private static boolean isSystemIndex(SortedMap indices } } - void createIndex(String index, boolean requireDataStream, TimeValue timeout, ActionListener listener) { - CreateIndexRequest createIndexRequest = new CreateIndexRequest(); - createIndexRequest.index(index); - createIndexRequest.requireDataStream(requireDataStream); - createIndexRequest.cause("auto(bulk api)"); - createIndexRequest.masterNodeTimeout(timeout); + void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { client.execute(AutoCreateAction.INSTANCE, createIndexRequest, listener); } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java index 83d331d2e4aa1..f0f950ca324bf 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportSimulateBulkAction.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; import org.elasticsearch.action.support.ActionFilters; @@ -72,7 +73,7 @@ protected void createMissingIndicesAndIndexData( BulkRequest bulkRequest, Executor executor, ActionListener listener, - Map indicesToAutoCreate, + Map indicesToAutoCreate, Set dataStreamsToRollover, Set failureStoresToBeRolledOver, long startTime diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 873c644725aba..399a4ad526537 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -178,7 +178,7 @@ public void sendClearAllScrollContexts(Transport.Connection connection, final Ac transportService.sendRequest( connection, CLEAR_SCROLL_CONTEXTS_ACTION_NAME, - TransportRequest.Empty.INSTANCE, + new ClearScrollContextsRequest(), TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>( listener, @@ -369,6 +369,14 @@ public ShardSearchContextId id() { } + private static class ClearScrollContextsRequest extends TransportRequest { + ClearScrollContextsRequest() {} + + ClearScrollContextsRequest(StreamInput in) throws IOException { + super(in); + } + } + static class SearchFreeContextRequest extends ScrollFreeContextRequest implements IndicesRequest { private final OriginalIndices originalIndices; @@ -456,7 +464,7 @@ public static void registerRequestHandler( transportService.registerRequestHandler( CLEAR_SCROLL_CONTEXTS_ACTION_NAME, transportService.getThreadPool().generic(), - TransportRequest.Empty::new, + ClearScrollContextsRequest::new, instrumentedHandler(CLEAR_SCROLL_CONTEXTS_ACTION_METRIC, transportService, searchTransportMetrics, (request, channel, task) -> { searchService.freeAllScrollContexts(); channel.sendResponse(TransportResponse.Empty.INSTANCE); diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java index 626cdb8046f53..d8628db4047e6 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesRequest.java @@ -10,54 +10,37 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; public abstract class BaseNodesRequest> extends ActionRequest { /** - * the list of nodesIds that will be used to resolve this request and {@link #concreteNodes} - * will be populated. Note that if {@link #concreteNodes} is not null, it will be used and nodeIds - * will be ignored. - * - * See {@link DiscoveryNodes#resolveNodes} for a full description of the options. - * - * TODO: we can get rid of this and resolve it to concrete nodes in the rest layer + * Sequence of node specifications that describe the nodes that this request should target. See {@link DiscoveryNodes#resolveNodes} for + * a full description of the options. If set, {@link #concreteNodes} is {@code null} and ignored. **/ - private String[] nodesIds; + private final String[] nodesIds; /** - * once {@link #nodesIds} are resolved this will contain the concrete nodes that are part of this request. If set, {@link #nodesIds} - * will be ignored and this will be used. - * */ - private DiscoveryNode[] concreteNodes; + * The exact nodes that this request should target. If set, {@link #nodesIds} is {@code null} and ignored. + **/ + private final DiscoveryNode[] concreteNodes; + @Nullable // if no timeout private TimeValue timeout; - /** - * @deprecated {@link BaseNodesRequest} derivatives are quite heavyweight and should never need sending over the wire. Do not include - * the full top-level request directly in the node-level requests. Instead, copy the needed fields over to a dedicated node-level - * request. - * - * @see #100878 - */ - @Deprecated(forRemoval = true) - protected BaseNodesRequest(StreamInput in) throws IOException { - // A bare `BaseNodesRequest` is never sent over the wire, but several implementations send the full top-level request to each node - // (wrapped up in another request). They shouldn't, but until we fix that we must keep this. See #100878. - super(in); - nodesIds = in.readStringArray(); - concreteNodes = in.readOptionalArray(DiscoveryNode::new, DiscoveryNode[]::new); - timeout = in.readOptionalTimeValue(); - } - - protected BaseNodesRequest(String... nodesIds) { + protected BaseNodesRequest(String[] nodesIds) { this.nodesIds = nodesIds; + this.concreteNodes = null; } protected BaseNodesRequest(DiscoveryNode... concreteNodes) { @@ -69,12 +52,6 @@ public final String[] nodesIds() { return nodesIds; } - @SuppressWarnings("unchecked") - public final Request nodesIds(String... nodesIds) { - this.nodesIds = nodesIds; - return (Request) this; - } - public TimeValue timeout() { return this.timeout; } @@ -85,26 +62,26 @@ public final Request timeout(TimeValue timeout) { return (Request) this; } - public DiscoveryNode[] concreteNodes() { - return concreteNodes; - } - - public void setConcreteNodes(DiscoveryNode[] concreteNodes) { - this.concreteNodes = concreteNodes; - } - @Override public ActionRequestValidationException validate() { return null; } @Override - public void writeTo(StreamOutput out) throws IOException { - // A bare `BaseNodesRequest` is never sent over the wire, but several implementations send the full top-level request to each node - // (wrapped up in another request). They shouldn't, but until we fix that we must keep this. See #100878. - super.writeTo(out); - out.writeStringArrayNullable(nodesIds); - out.writeOptionalArray(concreteNodes); - out.writeOptionalTimeValue(timeout); + public final void writeTo(StreamOutput out) throws IOException { + // `BaseNodesRequest` is rather heavyweight, especially all those `DiscoveryNodes` objects in larger clusters, and there is no need + // to send it out over the wire. Use a dedicated transport request just for the bits you need. + TransportAction.localOnly(); + } + + /** + * @return the nodes to which this request should fan out. + */ + DiscoveryNode[] resolveNodes(ClusterState clusterState) { + assert nodesIds == null || concreteNodes == null; + return Objects.requireNonNullElseGet( + concreteNodes, + () -> Arrays.stream(clusterState.nodes().resolveNodes(nodesIds)).map(clusterState.nodes()::get).toArray(DiscoveryNode[]::new) + ); } } diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java index ac193601212c1..3b0f246d8f30e 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponse.java @@ -11,8 +11,8 @@ import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.common.collect.Iterators; -import org.elasticsearch.common.xcontent.ChunkedToXContent; import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.common.xcontent.ChunkedToXContentObject; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.xcontent.ToXContent; @@ -21,7 +21,7 @@ public abstract class BaseNodesXContentResponse extends BaseNodesResponse implements - ChunkedToXContent { + ChunkedToXContentObject { protected BaseNodesXContentResponse(ClusterName clusterName, List nodes, List failures) { super(clusterName, nodes, failures); diff --git a/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java index bcbbe64a03d5e..1de7d3c0d93c7 100644 --- a/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/support/nodes/NodesOperationRequestBuilder.java @@ -24,12 +24,6 @@ protected NodesOperationRequestBuilder(ElasticsearchClient client, ActionType listener) { // coordination can run on SAME because it's only O(#nodes) work - if (request.concreteNodes() == null) { - resolveRequest(request, clusterService.state()); - assert request.concreteNodes() != null; - } + + final var concreteNodes = Objects.requireNonNull(resolveRequest(request, clusterService.state())); new CancellableFanOut, Exception>>() { - final ArrayList responses = new ArrayList<>(request.concreteNodes().length); + final ArrayList responses = new ArrayList<>(concreteNodes.length); final ArrayList exceptions = new ArrayList<>(0); final TransportRequestOptions transportRequestOptions = TransportRequestOptions.timeout(request.timeout()); @@ -172,7 +174,7 @@ public String toString() { } }.run( task, - Iterators.forArray(request.concreteNodes()), + Iterators.forArray(concreteNodes), new ThreadedActionListener<>(finalExecutor, listener.delegateFailureAndWrap((l, c) -> c.accept(l))) ); } @@ -235,10 +237,8 @@ protected void nodeOperationAsync(NodeRequest request, Task task, ActionListener * Resolves node ids to concrete nodes of the incoming request. * NB: if the request's nodeIds() returns nothing, then the request will be sent to ALL known nodes in the cluster. */ - protected void resolveRequest(NodesRequest request, ClusterState clusterState) { - assert request.concreteNodes() == null : "request concreteNodes shouldn't be set"; - String[] nodesIds = clusterState.nodes().resolveNodes(request.nodesIds()); - request.setConcreteNodes(Arrays.stream(nodesIds).map(clusterState.nodes()::get).toArray(DiscoveryNode[]::new)); + protected DiscoveryNode[] resolveRequest(NodesRequest request, ClusterState clusterState) { + return request.resolveNodes(clusterState); } class NodeTransportHandler implements TransportRequestHandler { @@ -251,4 +251,42 @@ public void messageReceived(NodeRequest request, TransportChannel channel, Task } } + /** + * Some {@link TransportNodesAction} implementations send the whole top-level request out to each individual node. However, the + * top-level request contains a lot of unnecessary junk, particularly the heavyweight {@link DiscoveryNode} instances, so we are + * migrating away from this practice. This method allows to skip over the unnecessary data received from an older node. + * + * @see #100878 + * @param fixVersion The {@link TransportVersion} in which the request representation was fixed, so no skipping is needed. + * @param in The {@link StreamInput} in which to skip the unneeded data. + */ + @UpdateForV9 // no longer necessary in v9 + public static void skipLegacyNodesRequestHeader(TransportVersion fixVersion, StreamInput in) throws IOException { + if (in.getTransportVersion().before(fixVersion)) { + TaskId.readFromStream(in); + in.readStringArray(); + in.readOptionalArray(DiscoveryNode::new, DiscoveryNode[]::new); + in.readOptionalTimeValue(); + } + } + + /** + * Some {@link TransportNodesAction} implementations send the whole top-level request out to each individual node. However, the + * top-level request contains a lot of unnecessary junk, particularly the heavyweight {@link DiscoveryNode} instances, so we are + * migrating away from this practice. This method allows to send a well-formed, but empty, header to older nodes that require it. + * + * @see #100878 + * @param fixVersion The {@link TransportVersion} in which the request representation was fixed, so no skipping is needed. + * @param out The {@link StreamOutput} to which to send the dummy data. + */ + @UpdateForV9 // no longer necessary in v9 + public static void sendLegacyNodesRequestHeader(TransportVersion fixVersion, StreamOutput out) throws IOException { + if (out.getTransportVersion().before(fixVersion)) { + TaskId.EMPTY_TASK_ID.writeTo(out); + out.writeStringArray(Strings.EMPTY_ARRAY); + out.writeOptionalArray(null); + out.writeOptionalTimeValue(null); + } + } + } diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 36b6cc6aa9964..2cd5258bf4376 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -30,7 +30,6 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ObjectParser; @@ -42,7 +41,6 @@ import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import static org.elasticsearch.action.ValidateActions.addValidationError; @@ -269,165 +267,6 @@ public UpdateRequest script(Script script) { return this; } - /** - * @deprecated Use {@link #script()} instead - */ - @Deprecated - public String scriptString() { - return this.script == null ? null : this.script.getIdOrCode(); - } - - /** - * @deprecated Use {@link #script()} instead - */ - @Deprecated - public ScriptType scriptType() { - return this.script == null ? null : this.script.getType(); - } - - /** - * @deprecated Use {@link #script()} instead - */ - @Deprecated - public Map scriptParams() { - return this.script == null ? null : this.script.getParams(); - } - - /** - * The script to execute. Note, make sure not to send different script each - * times and instead use script params if possible with the same - * (automatically compiled) script. - * - * @deprecated Use {@link #script(Script)} instead - */ - @Deprecated - public UpdateRequest script(String script, ScriptType scriptType) { - updateOrCreateScript(script, scriptType, null, null); - return this; - } - - /** - * The script to execute. Note, make sure not to send different script each - * times and instead use script params if possible with the same - * (automatically compiled) script. - * - * @deprecated Use {@link #script(Script)} instead - */ - @Deprecated - public UpdateRequest script(String script) { - updateOrCreateScript(script, ScriptType.INLINE, null, null); - return this; - } - - /** - * The language of the script to execute. - * - * @deprecated Use {@link #script(Script)} instead - */ - @Deprecated - public UpdateRequest scriptLang(String scriptLang) { - updateOrCreateScript(null, null, scriptLang, null); - return this; - } - - /** - * @deprecated Use {@link #script()} instead - */ - @Deprecated - public String scriptLang() { - return script == null ? null : script.getLang(); - } - - /** - * Add a script parameter. - * - * @deprecated Use {@link #script(Script)} instead - */ - @Deprecated - public UpdateRequest addScriptParam(String name, Object value) { - Script script = script(); - if (script == null) { - HashMap scriptParams = new HashMap<>(); - scriptParams.put(name, value); - updateOrCreateScript(null, null, null, scriptParams); - } else { - Map scriptParams = script.getParams(); - if (scriptParams == null) { - scriptParams = new HashMap<>(); - scriptParams.put(name, value); - updateOrCreateScript(null, null, null, scriptParams); - } else { - scriptParams.put(name, value); - } - } - return this; - } - - /** - * Sets the script parameters to use with the script. - * - * @deprecated Use {@link #script(Script)} instead - */ - @Deprecated - public UpdateRequest scriptParams(Map scriptParams) { - updateOrCreateScript(null, null, null, scriptParams); - return this; - } - - private void updateOrCreateScript(String scriptContent, ScriptType type, String lang, Map params) { - Script script = script(); - if (script == null) { - script = new Script(type == null ? ScriptType.INLINE : type, lang, scriptContent == null ? "" : scriptContent, params); - } else { - String newScriptContent = scriptContent == null ? script.getIdOrCode() : scriptContent; - ScriptType newScriptType = type == null ? script.getType() : type; - String newScriptLang = lang == null ? script.getLang() : lang; - Map newScriptParams = params == null ? script.getParams() : params; - script = new Script(newScriptType, newScriptLang, newScriptContent, newScriptParams); - } - script(script); - } - - /** - * The script to execute. Note, make sure not to send different script each - * times and instead use script params if possible with the same - * (automatically compiled) script. - * - * @deprecated Use {@link #script(Script)} instead - */ - @Deprecated - public UpdateRequest script(String script, ScriptType scriptType, @Nullable Map scriptParams) { - this.script = new Script(scriptType, Script.DEFAULT_SCRIPT_LANG, script, scriptParams); - return this; - } - - /** - * The script to execute. Note, make sure not to send different script each - * times and instead use script params if possible with the same - * (automatically compiled) script. - * - * @param script - * The script to execute - * @param scriptLang - * The script language - * @param scriptType - * The script type - * @param scriptParams - * The script parameters - * - * @deprecated Use {@link #script(Script)} instead - */ - @Deprecated - public UpdateRequest script( - String script, - @Nullable String scriptLang, - ScriptType scriptType, - @Nullable Map scriptParams - ) { - this.script = new Script(scriptType, scriptLang, script, scriptParams); - return this; - } - /** * Indicate that _source should be returned with every hit, with an * "include" and/or "exclude" set which can include simple wildcard diff --git a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java index 87e1489b076c1..4e42de57d08d3 100644 --- a/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java +++ b/server/src/main/java/org/elasticsearch/client/internal/ClusterAdminClient.java @@ -208,7 +208,7 @@ public void nodesInfo(final NodesInfoRequest request, final ActionListener listener) { @@ -228,7 +228,7 @@ public void nodesStats(final NodesStatsRequest request, final ActionListener nodesCapabilities(final NodesCapabilitiesRequest request) { diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java index 60140e2a08714..29933ad20ef10 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterModule.java @@ -157,6 +157,7 @@ public ClusterModule( snapshotsInfoService, shardRoutingRoleStrategy ); + this.allocationService.addAllocFailuresResetListenerTo(clusterService); this.metadataDeleteIndexService = new MetadataDeleteIndexService(settings, clusterService, allocationService); this.allocationStatsService = new AllocationStatsService(clusterService, clusterInfoService, shardsAllocator, writeLoadForecaster); this.telemetryProvider = telemetryProvider; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java index 95f4964035887..2f604f1b95974 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/Coordinator.java @@ -72,7 +72,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool.Names; import org.elasticsearch.transport.NodeDisconnectedException; -import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse.Empty; import org.elasticsearch.transport.TransportResponseHandler; @@ -759,7 +758,7 @@ private void sendJoinPing(DiscoveryNode discoveryNode, TransportRequestOptions.T transportService.sendRequest( discoveryNode, JoinHelper.JOIN_PING_ACTION_NAME, - TransportRequest.Empty.INSTANCE, + new JoinHelper.JoinPingRequest(), TransportRequestOptions.of(null, channelType), TransportResponseHandler.empty(clusterCoordinationExecutor, listener.delegateResponse((l, e) -> { logger.warn(() -> format("failed to ping joining node [%s] on channel type [%s]", discoveryNode, channelType), e); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java index 059400ad81cfb..05dbc66c95971 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinHelper.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.Releasable; @@ -47,6 +48,7 @@ import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -154,7 +156,7 @@ public class JoinHelper { EsExecutors.DIRECT_EXECUTOR_SERVICE, false, false, - TransportRequest.Empty::new, + JoinPingRequest::new, (request, channel, task) -> channel.sendResponse(Empty.INSTANCE) ); } @@ -606,4 +608,12 @@ private static class PendingJoinInfo { static final String PENDING_JOIN_WAITING_STATE = "waiting to receive cluster state"; static final String PENDING_JOIN_CONNECT_FAILED = "failed to connect"; static final String PENDING_JOIN_FAILED = "failed"; + + static class JoinPingRequest extends TransportRequest { + JoinPingRequest() {} + + JoinPingRequest(StreamInput in) throws IOException { + super(in); + } + } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java index 6ba35d6aec25a..c20a3d64b5543 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/JoinValidationService.java @@ -37,7 +37,6 @@ import org.elasticsearch.transport.BytesTransportRequest; import org.elasticsearch.transport.NodeNotConnectedException; import org.elasticsearch.transport.Transport; -import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.transport.TransportRequestOptions; import org.elasticsearch.transport.TransportResponse; import org.elasticsearch.transport.TransportResponseHandler; @@ -207,7 +206,7 @@ private void legacyValidateJoin(DiscoveryNode discoveryNode, ActionListener Releasables.close(releasable)) ); success = true; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index bf1d9462ab89f..03b23c462ecec 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -581,23 +581,13 @@ public DataStream removeFailureStoreIndex(Index index) { ); } - // TODO: When failure stores are lazily created, this wont necessarily be required anymore. We can remove the failure store write - // index as long as we mark the data stream to lazily rollover the failure store with no conditions on its next write - if (failureIndices.indices.size() == (failureIndexPosition + 1)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "cannot remove backing index [%s] of data stream [%s] because it is the write index of the failure store", - index.getName(), - name - ) - ); - } - + // If this is the write index, we're marking the failure store for lazy rollover, to make sure a new write index gets created on the + // next write. We do this regardless of whether it's the last index in the failure store or not. + boolean rolloverOnWrite = failureIndices.indices.size() == (failureIndexPosition + 1); List updatedFailureIndices = new ArrayList<>(failureIndices.indices); updatedFailureIndices.remove(index); assert updatedFailureIndices.size() == failureIndices.indices.size() - 1; - return copy().setFailureIndices(failureIndices.copy().setIndices(updatedFailureIndices).build()) + return copy().setFailureIndices(failureIndices.copy().setIndices(updatedFailureIndices).setRolloverOnWrite(rolloverOnWrite).build()) .setGeneration(generation + 1) .build(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 8bc8f9d96bf24..459c6c6ec733e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -422,7 +422,7 @@ private static void resolveIndicesForDataStream(Context context, DataStream data } } } - if (shouldIncludeFailureIndices(context.getOptions(), dataStream)) { + if (shouldIncludeFailureIndices(context.getOptions())) { // We short-circuit here, if failure indices are not allowed and they can be skipped if (context.getOptions().allowFailureIndices() || context.getOptions().ignoreUnavailable() == false) { for (Index index : dataStream.getFailureIndices().getIndices()) { @@ -441,7 +441,7 @@ private static void resolveWriteIndexForDataStreams(Context context, DataStream concreteIndicesResult.add(writeIndex); } } - if (shouldIncludeFailureIndices(context.getOptions(), dataStream)) { + if (shouldIncludeFailureIndices(context.getOptions())) { Index failureStoreWriteIndex = dataStream.getFailureStoreWriteIndex(); if (failureStoreWriteIndex != null && addIndex(failureStoreWriteIndex, null, context)) { if (context.options.allowFailureIndices() == false) { @@ -456,10 +456,9 @@ private static boolean shouldIncludeRegularIndices(IndicesOptions indicesOptions return DataStream.isFailureStoreFeatureFlagEnabled() == false || indicesOptions.includeRegularIndices(); } - private static boolean shouldIncludeFailureIndices(IndicesOptions indicesOptions, DataStream dataStream) { - return DataStream.isFailureStoreFeatureFlagEnabled() - && indicesOptions.includeFailureIndices() - && dataStream.isFailureStoreEnabled(); + private static boolean shouldIncludeFailureIndices(IndicesOptions indicesOptions) { + // We return failure indices regardless of whether the data stream actually has the `failureStoreEnabled` flag set to true. + return DataStream.isFailureStoreFeatureFlagEnabled() && indicesOptions.includeFailureIndices(); } private static boolean resolvesToMoreThanOneIndex(IndexAbstraction indexAbstraction, Context context) { @@ -469,7 +468,7 @@ private static boolean resolvesToMoreThanOneIndex(IndexAbstraction indexAbstract if (shouldIncludeRegularIndices(context.getOptions())) { count += dataStream.getIndices().size(); } - if (shouldIncludeFailureIndices(context.getOptions(), dataStream)) { + if (shouldIncludeFailureIndices(context.getOptions())) { count += dataStream.getFailureIndices().getIndices().size(); } return count > 1; @@ -1426,8 +1425,7 @@ private static Stream expandToOpenClosed(Context context, Stream rerouteListener + ActionListener rerouteListener, + boolean initializeFailureStore ) throws Exception { - return createDataStream(metadataCreateIndexService, clusterService.getSettings(), current, isDslOnlyMode, request, rerouteListener); + return createDataStream( + metadataCreateIndexService, + clusterService.getSettings(), + current, + isDslOnlyMode, + request, + rerouteListener, + initializeFailureStore + ); } public static final class CreateDataStreamClusterStateUpdateRequest extends ClusterStateUpdateRequest< @@ -194,7 +198,8 @@ static ClusterState createDataStream( ClusterState currentState, boolean isDslOnlyMode, CreateDataStreamClusterStateUpdateRequest request, - ActionListener rerouteListener + ActionListener rerouteListener, + boolean initializeFailureStore ) throws Exception { return createDataStream( metadataCreateIndexService, @@ -204,7 +209,8 @@ static ClusterState createDataStream( request, List.of(), null, - rerouteListener + rerouteListener, + initializeFailureStore ); } @@ -212,11 +218,12 @@ static ClusterState createDataStream( * Creates a data stream with the specified request, backing indices and write index. * * @param metadataCreateIndexService Used if a new write index must be created - * @param currentState Cluster state - * @param request The create data stream request - * @param backingIndices List of backing indices. May be empty - * @param writeIndex Write index for the data stream. If null, a new write index will be created. - * @return Cluster state containing the new data stream + * @param currentState Cluster state + * @param request The create data stream request + * @param backingIndices List of backing indices. May be empty + * @param writeIndex Write index for the data stream. If null, a new write index will be created. + * @param initializeFailureStore Whether the failure store should be initialized + * @return Cluster state containing the new data stream */ static ClusterState createDataStream( MetadataCreateIndexService metadataCreateIndexService, @@ -226,7 +233,8 @@ static ClusterState createDataStream( CreateDataStreamClusterStateUpdateRequest request, List backingIndices, IndexMetadata writeIndex, - ActionListener rerouteListener + ActionListener rerouteListener, + boolean initializeFailureStore ) throws Exception { String dataStreamName = request.name; SystemDataStreamDescriptor systemDataStreamDescriptor = request.getSystemDataStreamDescriptor(); @@ -274,7 +282,7 @@ static ClusterState createDataStream( // If we need to create a failure store, do so first. Do not reroute during the creation since we will do // that as part of creating the backing index if required. IndexMetadata failureStoreIndex = null; - if (template.getDataStreamTemplate().hasFailureStore()) { + if (template.getDataStreamTemplate().hasFailureStore() && initializeFailureStore) { if (isSystem) { throw new IllegalArgumentException("Failure stores are not supported on system data streams"); } @@ -312,7 +320,8 @@ static ClusterState createDataStream( } assert writeIndex != null; assert writeIndex.mapping() != null : "no mapping found for backing index [" + writeIndex.getIndex().getName() + "]"; - assert template.getDataStreamTemplate().hasFailureStore() == false || failureStoreIndex != null; + assert template.getDataStreamTemplate().hasFailureStore() == false || initializeFailureStore == false || failureStoreIndex != null + : "failure store should have an initial index"; assert failureStoreIndex == null || failureStoreIndex.mapping() != null : "no mapping found for failure store [" + failureStoreIndex.getIndex().getName() + "]"; @@ -328,19 +337,20 @@ static ClusterState createDataStream( List failureIndices = failureStoreIndex == null ? List.of() : List.of(failureStoreIndex.getIndex()); DataStream newDataStream = new DataStream( dataStreamName, - dsBackingIndices, initialGeneration, template.metadata() != null ? Map.copyOf(template.metadata()) : null, hidden, false, isSystem, + System::currentTimeMillis, template.getDataStreamTemplate().isAllowCustomRouting(), indexMode, lifecycle == null && isDslOnlyMode ? DataStreamLifecycle.DEFAULT : lifecycle, template.getDataStreamTemplate().hasFailureStore(), - failureIndices, - false, - null + new DataStream.DataStreamIndices(DataStream.BACKING_INDEX_PREFIX, dsBackingIndices, false, null), + // If the failure store shouldn't be initialized on data stream creation, we're marking it for "lazy rollover", which will + // initialize the failure store on first write. + new DataStream.DataStreamIndices(DataStream.FAILURE_STORE_PREFIX, failureIndices, initializeFailureStore == false, null) ); Metadata.Builder builder = Metadata.builder(currentState.metadata()).put(newDataStream); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index affc331c5ab49..6d99874fd2edb 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -805,9 +805,7 @@ static void validateLifecycle( ComposableIndexTemplate template, @Nullable DataStreamGlobalRetention globalRetention ) { - DataStreamLifecycle lifecycle = template.template() != null && template.template().lifecycle() != null - ? template.template().lifecycle() - : resolveLifecycle(template, metadata.componentTemplates()); + DataStreamLifecycle lifecycle = resolveLifecycle(template, metadata.componentTemplates()); if (lifecycle != null) { if (template.getDataStreamTemplate() == null) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java index 56170ffb16cd3..9dbbdd597a4ce 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamService.java @@ -165,7 +165,9 @@ static ClusterState migrateToDataStream( req, backingIndices, currentState.metadata().index(writeIndex), - listener + listener, + // No need to initialize the failure store when migrating to a data stream. + false ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java index 2345b935cfadf..508f8346a875d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ShutdownShardMigrationStatus.java @@ -89,6 +89,25 @@ public ShutdownShardMigrationStatus( ); } + public ShutdownShardMigrationStatus( + SingleNodeShutdownMetadata.Status status, + long startedShards, + long relocatingShards, + long initializingShards, + @Nullable String explanation, + @Nullable ShardAllocationDecision allocationDecision + ) { + this( + status, + startedShards, + relocatingShards, + initializingShards, + startedShards + relocatingShards + initializingShards, + explanation, + allocationDecision + ); + } + private ShutdownShardMigrationStatus( SingleNodeShutdownMetadata.Status status, long startedShards, @@ -140,6 +159,10 @@ public SingleNodeShutdownMetadata.Status getStatus() { return status; } + public ShardAllocationDecision getAllocationDecision() { + return allocationDecision; + } + @Override public Iterator toXContentChunked(ToXContent.Params params) { return Iterators.concat( diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java index 0b3cadb6e187c..7a57df310252b 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/RoutingNodes.java @@ -1276,6 +1276,15 @@ private void ensureMutable() { } } + public boolean hasAllocationFailures() { + return unassignedShards.stream().anyMatch((shardRouting -> { + if (shardRouting.unassignedInfo() == null) { + return false; + } + return shardRouting.unassignedInfo().failedAllocations() > 0; + })); + } + public void resetFailedCounter(RoutingChangesObserver routingChangesObserver) { final var unassignedIterator = unassigned().iterator(); while (unassignedIterator.hasNext()) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java index d7bcacd3a0cde..61d44f45e01ff 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocateUnassignedDecision.java @@ -34,9 +34,9 @@ public class AllocateUnassignedDecision extends AbstractAllocationDecision { /** a constant representing a shard decision where no decision was taken */ public static final AllocateUnassignedDecision NOT_TAKEN = new AllocateUnassignedDecision( - AllocationStatus.NO_ATTEMPT, null, null, + AllocationStatus.NO_ATTEMPT, null, false, 0L, @@ -51,23 +51,23 @@ public class AllocateUnassignedDecision extends AbstractAllocationDecision { Map cachedDecisions = new EnumMap<>(AllocationStatus.class); cachedDecisions.put( AllocationStatus.FETCHING_SHARD_DATA, - new AllocateUnassignedDecision(AllocationStatus.FETCHING_SHARD_DATA, null, null, null, false, 0L, 0L) + new AllocateUnassignedDecision(null, null, AllocationStatus.FETCHING_SHARD_DATA, null, false, 0L, 0L) ); cachedDecisions.put( AllocationStatus.NO_VALID_SHARD_COPY, - new AllocateUnassignedDecision(AllocationStatus.NO_VALID_SHARD_COPY, null, null, null, false, 0L, 0L) + new AllocateUnassignedDecision(null, null, AllocationStatus.NO_VALID_SHARD_COPY, null, false, 0L, 0L) ); cachedDecisions.put( AllocationStatus.DECIDERS_NO, - new AllocateUnassignedDecision(AllocationStatus.DECIDERS_NO, null, null, null, false, 0L, 0L) + new AllocateUnassignedDecision(null, null, AllocationStatus.DECIDERS_NO, null, false, 0L, 0L) ); cachedDecisions.put( AllocationStatus.DECIDERS_THROTTLED, - new AllocateUnassignedDecision(AllocationStatus.DECIDERS_THROTTLED, null, null, null, false, 0L, 0L) + new AllocateUnassignedDecision(null, null, AllocationStatus.DECIDERS_THROTTLED, null, false, 0L, 0L) ); cachedDecisions.put( AllocationStatus.DELAYED_ALLOCATION, - new AllocateUnassignedDecision(AllocationStatus.DELAYED_ALLOCATION, null, null, null, false, 0L, 0L) + new AllocateUnassignedDecision(null, null, AllocationStatus.DELAYED_ALLOCATION, null, false, 0L, 0L) ); CACHED_DECISIONS = Collections.unmodifiableMap(cachedDecisions); } @@ -81,10 +81,10 @@ public class AllocateUnassignedDecision extends AbstractAllocationDecision { private final long configuredDelayInMillis; private AllocateUnassignedDecision( - AllocationStatus allocationStatus, DiscoveryNode assignedNode, - String allocationId, List nodeDecisions, + AllocationStatus allocationStatus, + String allocationId, boolean reuseStore, long remainingDelayInMillis, long configuredDelayInMillis @@ -145,7 +145,7 @@ private static AllocateUnassignedDecision no( long totalDelay ) { if (decisions != null) { - return new AllocateUnassignedDecision(allocationStatus, null, null, decisions, reuseStore, remainingDelay, totalDelay); + return new AllocateUnassignedDecision(null, decisions, allocationStatus, null, reuseStore, remainingDelay, totalDelay); } else { return getCachedDecision(allocationStatus); } @@ -157,7 +157,7 @@ private static AllocateUnassignedDecision no( */ public static AllocateUnassignedDecision throttle(@Nullable List decisions) { if (decisions != null) { - return new AllocateUnassignedDecision(AllocationStatus.DECIDERS_THROTTLED, null, null, decisions, false, 0L, 0L); + return new AllocateUnassignedDecision(null, decisions, AllocationStatus.DECIDERS_THROTTLED, null, false, 0L, 0L); } else { return getCachedDecision(AllocationStatus.DECIDERS_THROTTLED); } @@ -174,7 +174,7 @@ public static AllocateUnassignedDecision yes( @Nullable List decisions, boolean reuseStore ) { - return new AllocateUnassignedDecision(null, assignedNode, allocationId, decisions, reuseStore, 0L, 0L); + return new AllocateUnassignedDecision(assignedNode, decisions, null, allocationId, reuseStore, 0L, 0L); } /** @@ -187,7 +187,7 @@ public static AllocateUnassignedDecision fromDecision( ) { final Type decisionType = decision.type(); AllocationStatus allocationStatus = decisionType != Type.YES ? AllocationStatus.fromDecision(decisionType) : null; - return new AllocateUnassignedDecision(allocationStatus, assignedNode, null, nodeDecisions, false, 0L, 0L); + return new AllocateUnassignedDecision(assignedNode, nodeDecisions, allocationStatus, null, false, 0L, 0L); } private static AllocateUnassignedDecision getCachedDecision(AllocationStatus allocationStatus) { diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java index 436399a02005f..17bbc8f20793d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/AllocationService.java @@ -35,6 +35,9 @@ import org.elasticsearch.cluster.routing.allocation.command.AllocationCommands; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.Decision; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.cluster.service.MasterService; +import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.common.logging.ESLogMessage; @@ -559,6 +562,25 @@ private void allocateExistingUnassignedShards(RoutingAllocation allocation) { } } + /** + * Creates a cluster state listener that resets allocation failures. For example, reset when a new node joins a cluster. Resetting + * counter on new node join covers a variety of use cases, such as rolling update, version change, node restarts. + */ + public void addAllocFailuresResetListenerTo(ClusterService clusterService) { + // batched cluster update executor, runs reroute once per batch + // set retryFailed=true to trigger failures reset during reroute + var taskQueue = clusterService.createTaskQueue("reset-allocation-failures", Priority.NORMAL, (batchCtx) -> { + batchCtx.taskContexts().forEach((taskCtx) -> taskCtx.success(() -> {})); + return reroute(batchCtx.initialState(), new AllocationCommands(), false, true, false, ActionListener.noop()).clusterState(); + }); + + clusterService.addListener((changeEvent) -> { + if (changeEvent.nodesAdded() && changeEvent.state().getRoutingNodes().hasAllocationFailures()) { + taskQueue.submitTask("reset-allocation-failures", (e) -> { assert MasterService.isPublishFailureException(e); }, null); + } + }); + } + private static void disassociateDeadNodes(RoutingAllocation allocation) { for (Iterator it = allocation.routingNodes().mutableIterator(); it.hasNext();) { RoutingNode node = it.next(); diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java index 3819805316f26..692bf05a9c695 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/MoveDecision.java @@ -31,24 +31,24 @@ public final class MoveDecision extends AbstractAllocationDecision { public static final MoveDecision NOT_TAKEN = new MoveDecision(null, null, AllocationDecision.NO_ATTEMPT, null, null, 0); /** cached decisions so we don't have to recreate objects for common decisions when not in explain mode. */ private static final MoveDecision CACHED_STAY_DECISION = new MoveDecision( - Decision.YES, null, - AllocationDecision.NO_ATTEMPT, null, + AllocationDecision.NO_ATTEMPT, + Decision.YES, null, 0 ); private static final MoveDecision CACHED_CANNOT_MOVE_DECISION = new MoveDecision( - Decision.NO, null, - AllocationDecision.NO, null, + AllocationDecision.NO, + Decision.NO, null, 0 ); @Nullable - AllocationDecision allocationDecision; + private final AllocationDecision canMoveDecision; @Nullable private final Decision canRemainDecision; @Nullable @@ -56,15 +56,15 @@ public final class MoveDecision extends AbstractAllocationDecision { private final int currentNodeRanking; private MoveDecision( + DiscoveryNode targetNode, + List nodeDecisions, + AllocationDecision canMoveDecision, Decision canRemainDecision, Decision clusterRebalanceDecision, - AllocationDecision allocationDecision, - DiscoveryNode assignedNode, - List nodeDecisions, int currentNodeRanking ) { - super(assignedNode, nodeDecisions); - this.allocationDecision = allocationDecision; + super(targetNode, nodeDecisions); + this.canMoveDecision = canMoveDecision; this.canRemainDecision = canRemainDecision; this.clusterRebalanceDecision = clusterRebalanceDecision; this.currentNodeRanking = currentNodeRanking; @@ -72,7 +72,7 @@ private MoveDecision( public MoveDecision(StreamInput in) throws IOException { super(in); - allocationDecision = in.readOptionalWriteable(AllocationDecision::readFrom); + canMoveDecision = in.readOptionalWriteable(AllocationDecision::readFrom); canRemainDecision = in.readOptionalWriteable(Decision::readFrom); clusterRebalanceDecision = in.readOptionalWriteable(Decision::readFrom); currentNodeRanking = in.readVInt(); @@ -81,7 +81,7 @@ public MoveDecision(StreamInput in) throws IOException { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeOptionalWriteable(allocationDecision); + out.writeOptionalWriteable(canMoveDecision); out.writeOptionalWriteable(canRemainDecision); out.writeOptionalWriteable(clusterRebalanceDecision); out.writeVInt(currentNodeRanking); @@ -91,63 +91,52 @@ public void writeTo(StreamOutput out) throws IOException { * Creates a move decision for the shard being able to remain on its current node, so the shard won't * be forced to move to another node. */ - public static MoveDecision stay(Decision canRemainDecision) { + public static MoveDecision remain(Decision canRemainDecision) { if (canRemainDecision == Decision.YES) { return CACHED_STAY_DECISION; } assert canRemainDecision.type() != Type.NO; - return new MoveDecision(canRemainDecision, null, AllocationDecision.NO_ATTEMPT, null, null, 0); + return new MoveDecision(null, null, AllocationDecision.NO_ATTEMPT, canRemainDecision, null, 0); } /** - * Creates a move decision for the shard not being allowed to remain on its current node. + * Creates a move decision for the shard. * * @param canRemainDecision the decision for whether the shard is allowed to remain on its current node - * @param allocationDecision the {@link AllocationDecision} for moving the shard to another node - * @param assignedNode the node where the shard should move to + * @param moveDecision the {@link AllocationDecision} for moving the shard to another node + * @param targetNode the node where the shard should move to * @param nodeDecisions the node-level decisions that comprised the final decision, non-null iff explain is true * @return the {@link MoveDecision} for moving the shard to another node */ - public static MoveDecision cannotRemain( + public static MoveDecision move( Decision canRemainDecision, - AllocationDecision allocationDecision, - DiscoveryNode assignedNode, - List nodeDecisions + AllocationDecision moveDecision, + @Nullable DiscoveryNode targetNode, + @Nullable List nodeDecisions ) { assert canRemainDecision != null; assert canRemainDecision.type() != Type.YES : "create decision with MoveDecision#stay instead"; - if (nodeDecisions == null && allocationDecision == AllocationDecision.NO) { + if (nodeDecisions == null && moveDecision == AllocationDecision.NO) { // the final decision is NO (no node to move the shard to) and we are not in explain mode, return a cached version return CACHED_CANNOT_MOVE_DECISION; } else { - assert ((assignedNode == null) == (allocationDecision != AllocationDecision.YES)); - return new MoveDecision(canRemainDecision, null, allocationDecision, assignedNode, nodeDecisions, 0); + assert ((targetNode == null) == (moveDecision != AllocationDecision.YES)); + return new MoveDecision(targetNode, nodeDecisions, moveDecision, canRemainDecision, null, 0); } } - /** - * Creates a move decision for when rebalancing the shard is not allowed. - */ - public static MoveDecision cannotRebalance( - Decision canRebalanceDecision, - AllocationDecision allocationDecision, - int currentNodeRanking, - List nodeDecisions - ) { - return new MoveDecision(null, canRebalanceDecision, allocationDecision, null, nodeDecisions, currentNodeRanking); - } - /** * Creates a decision for whether to move the shard to a different node to form a better cluster balance. */ public static MoveDecision rebalance( + Decision canRemainDecision, Decision canRebalanceDecision, - AllocationDecision allocationDecision, - @Nullable DiscoveryNode assignedNode, + AllocationDecision canMoveDecision, + @Nullable DiscoveryNode targetNode, int currentNodeRanking, List nodeDecisions ) { - return new MoveDecision(null, canRebalanceDecision, allocationDecision, assignedNode, nodeDecisions, currentNodeRanking); + return new MoveDecision(targetNode, nodeDecisions, canMoveDecision, canRemainDecision, canRebalanceDecision, currentNodeRanking); } @Override @@ -155,20 +144,6 @@ public boolean isDecisionTaken() { return canRemainDecision != null || clusterRebalanceDecision != null; } - /** - * Creates a new move decision from this decision, plus adding a remain decision. - */ - public MoveDecision withRemainDecision(Decision canRemainDecision) { - return new MoveDecision( - canRemainDecision, - clusterRebalanceDecision, - allocationDecision, - targetNode, - nodeDecisions, - currentNodeRanking - ); - } - /** * Returns {@code true} if the shard cannot remain on its current node and can be moved, * returns {@code false} otherwise. If {@link #isDecisionTaken()} returns {@code false}, @@ -176,7 +151,7 @@ public MoveDecision withRemainDecision(Decision canRemainDecision) { */ public boolean forceMove() { checkDecisionState(); - return canRemain() == false && allocationDecision == AllocationDecision.YES; + return canRemain() == false && canMoveDecision == AllocationDecision.YES; } /** @@ -228,7 +203,7 @@ public Decision getClusterRebalanceDecision() { */ @Nullable public AllocationDecision getAllocationDecision() { - return allocationDecision; + return canMoveDecision; } /** @@ -248,7 +223,7 @@ public String getExplanation() { checkDecisionState(); if (clusterRebalanceDecision != null) { // it was a decision to rebalance the shard, because the shard was allowed to remain on its current node - if (allocationDecision == AllocationDecision.AWAITING_INFO) { + if (canMoveDecision == AllocationDecision.AWAITING_INFO) { return Explanations.Rebalance.AWAITING_INFO; } return switch (clusterRebalanceDecision.type()) { @@ -258,11 +233,9 @@ public String getExplanation() { case THROTTLE -> Explanations.Rebalance.CLUSTER_THROTTLE; case YES -> { if (getTargetNode() != null) { - if (allocationDecision == AllocationDecision.THROTTLED) { - yield Explanations.Rebalance.NODE_THROTTLE; - } else { - yield Explanations.Rebalance.YES; - } + yield canMoveDecision == AllocationDecision.THROTTLED + ? Explanations.Rebalance.NODE_THROTTLE + : Explanations.Rebalance.YES; } else { yield Explanations.Rebalance.ALREADY_BALANCED; } @@ -271,13 +244,13 @@ public String getExplanation() { } else { // it was a decision to force move the shard assert canRemain() == false; - return switch (allocationDecision) { + return switch (canMoveDecision) { case YES -> Explanations.Move.YES; case THROTTLED -> Explanations.Move.THROTTLED; case NO -> Explanations.Move.NO; case WORSE_BALANCE, AWAITING_INFO, ALLOCATION_DELAYED, NO_VALID_SHARD_COPY, NO_ATTEMPT -> { - assert false : allocationDecision; - yield allocationDecision.toString(); + assert false : canMoveDecision; + yield canMoveDecision.toString(); } }; } @@ -308,7 +281,7 @@ public Iterator toXContentChunked(ToXContent.Params params } } if (clusterRebalanceDecision != null) { - builder.field("can_rebalance_to_other_node", allocationDecision); + builder.field("can_rebalance_to_other_node", canMoveDecision); builder.field("rebalance_explanation", getExplanation()); } else { builder.field("can_move_to_other_node", forceMove() ? "yes" : "no"); @@ -327,7 +300,7 @@ public boolean equals(Object other) { return false; } MoveDecision that = (MoveDecision) other; - return Objects.equals(allocationDecision, that.allocationDecision) + return Objects.equals(canMoveDecision, that.canMoveDecision) && Objects.equals(canRemainDecision, that.canRemainDecision) && Objects.equals(clusterRebalanceDecision, that.clusterRebalanceDecision) && currentNodeRanking == that.currentNodeRanking; @@ -335,7 +308,7 @@ public boolean equals(Object other) { @Override public int hashCode() { - return 31 * super.hashCode() + Objects.hash(allocationDecision, canRemainDecision, clusterRebalanceDecision, currentNodeRanking); + return 31 * super.hashCode() + Objects.hash(canMoveDecision, canRemainDecision, clusterRebalanceDecision, currentNodeRanking); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 2fca8895b011c..193a1558c857a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -62,34 +62,31 @@ import static org.elasticsearch.common.settings.ClusterSettings.createBuiltInClusterSettings; /** - * The {@link BalancedShardsAllocator} re-balances the nodes allocations - * within an cluster based on a {@link WeightFunction}. The clusters balance is defined by four parameters which can be set - * in the cluster update API that allows changes in real-time: - *
  • cluster.routing.allocation.balance.shard - The shard balance defines the weight factor - * for shards allocated on a {@link RoutingNode}
    • - *
  • cluster.routing.allocation.balance.index - The index balance defines a factor to the number - * of {@link org.elasticsearch.cluster.routing.ShardRouting}s per index allocated on a specific node
    • - *
  • cluster.routing.allocation.balance.threshold - A threshold to set the minimal optimization - * value of operations that should be performed
    • + * The {@link BalancedShardsAllocator} allocates and balances shards on the cluster nodes using {@link WeightFunction}. + * The balancing attempts to: + *
      + *
    • even shard count across nodes (weighted by cluster.routing.allocation.balance.shard)
      • + *
    • spread shards of the same index across different nodes (weighted by cluster.routing.allocation.balance.index)
      • + *
    • even write load of the data streams write indices across nodes (weighted by cluster.routing.allocation.balance.write_load)
      • + *
    • even disk usage across nodes (weighted by cluster.routing.allocation.balance.write_load)
      • *
    - *

    - * These parameters are combined in a {@link WeightFunction} that allows calculation of node weights which - * are used to re-balance shards based on global as well as per-index factors. + * The sensitivity of the algorithm is defined by cluster.routing.allocation.balance.threshold. + * Allocator takes into account constraints set by {@code AllocationDeciders} when allocating and balancing shards. */ public class BalancedShardsAllocator implements ShardsAllocator { private static final Logger logger = LogManager.getLogger(BalancedShardsAllocator.class); - public static final Setting INDEX_BALANCE_FACTOR_SETTING = Setting.floatSetting( - "cluster.routing.allocation.balance.index", - 0.55f, + public static final Setting SHARD_BALANCE_FACTOR_SETTING = Setting.floatSetting( + "cluster.routing.allocation.balance.shard", + 0.45f, 0.0f, Property.Dynamic, Property.NodeScope ); - public static final Setting SHARD_BALANCE_FACTOR_SETTING = Setting.floatSetting( - "cluster.routing.allocation.balance.shard", - 0.45f, + public static final Setting INDEX_BALANCE_FACTOR_SETTING = Setting.floatSetting( + "cluster.routing.allocation.balance.index", + 0.55f, 0.0f, Property.Dynamic, Property.NodeScope @@ -138,8 +135,8 @@ public BalancedShardsAllocator(ClusterSettings clusterSettings) { @Inject public BalancedShardsAllocator(ClusterSettings clusterSettings, WriteLoadForecaster writeLoadForecaster) { - clusterSettings.initializeAndWatch(INDEX_BALANCE_FACTOR_SETTING, value -> this.indexBalanceFactor = value); clusterSettings.initializeAndWatch(SHARD_BALANCE_FACTOR_SETTING, value -> this.shardBalanceFactor = value); + clusterSettings.initializeAndWatch(INDEX_BALANCE_FACTOR_SETTING, value -> this.indexBalanceFactor = value); clusterSettings.initializeAndWatch(WRITE_LOAD_BALANCE_FACTOR_SETTING, value -> this.writeLoadBalanceFactor = value); clusterSettings.initializeAndWatch(DISK_USAGE_BALANCE_FACTOR_SETTING, value -> this.diskUsageBalanceFactor = value); clusterSettings.initializeAndWatch(THRESHOLD_SETTING, value -> this.threshold = ensureValidThreshold(value)); @@ -179,8 +176,8 @@ public void allocate(RoutingAllocation allocation) { return; } final WeightFunction weightFunction = new WeightFunction( - indexBalanceFactor, shardBalanceFactor, + indexBalanceFactor, writeLoadBalanceFactor, diskUsageBalanceFactor ); @@ -193,8 +190,8 @@ public void allocate(RoutingAllocation allocation) { @Override public ShardAllocationDecision decideShardAllocation(final ShardRouting shard, final RoutingAllocation allocation) { WeightFunction weightFunction = new WeightFunction( - indexBalanceFactor, shardBalanceFactor, + indexBalanceFactor, writeLoadBalanceFactor, diskUsageBalanceFactor ); @@ -206,8 +203,7 @@ public ShardAllocationDecision decideShardAllocation(final ShardRouting shard, f } else { moveDecision = balancer.decideMove(shard); if (moveDecision.isDecisionTaken() && moveDecision.canRemain()) { - MoveDecision rebalanceDecision = balancer.decideRebalance(shard); - moveDecision = rebalanceDecision.withRemainDecision(moveDecision.getCanRemainDecision()); + moveDecision = balancer.decideRebalance(shard, moveDecision.getCanRemainDecision()); } } return new ShardAllocationDecision(allocateUnassignedDecision, moveDecision); @@ -293,8 +289,8 @@ private static class WeightFunction { private final float theta2; private final float theta3; - WeightFunction(float indexBalance, float shardBalance, float writeLoadBalance, float diskUsageBalance) { - float sum = indexBalance + shardBalance + writeLoadBalance + diskUsageBalance; + WeightFunction(float shardBalance, float indexBalance, float writeLoadBalance, float diskUsageBalance) { + float sum = shardBalance + indexBalance + writeLoadBalance + diskUsageBalance; if (sum <= 0.0f) { throw new IllegalArgumentException("Balance factors must sum to a value > 0 but was: " + sum); } @@ -523,7 +519,7 @@ private void balance() { * optimally balanced cluster. This method is invoked from the cluster allocation * explain API only. */ - private MoveDecision decideRebalance(final ShardRouting shard) { + private MoveDecision decideRebalance(final ShardRouting shard, Decision canRemain) { if (shard.started() == false) { // we can only rebalance started shards return MoveDecision.NOT_TAKEN; @@ -549,7 +545,7 @@ private MoveDecision decideRebalance(final ShardRouting shard) { final float currentWeight = weight.weight(this, currentNode, idxName); final AllocationDeciders deciders = allocation.deciders(); Type rebalanceDecisionType = Type.NO; - ModelNode assignedNode = null; + ModelNode targetNode = null; List> betterBalanceNodes = new ArrayList<>(); List> sameBalanceNodes = new ArrayList<>(); List> worseBalanceNodes = new ArrayList<>(); @@ -588,7 +584,7 @@ private MoveDecision decideRebalance(final ShardRouting shard) { // rebalance to the node, only will get overwritten if the decision here is to // THROTTLE and we get a decision with YES on another node rebalanceDecisionType = canAllocate.type(); - assignedNode = node; + targetNode = node; } } Tuple nodeResult = Tuple.tuple(node, canAllocate); @@ -626,15 +622,23 @@ private MoveDecision decideRebalance(final ShardRouting shard) { } if (canRebalance.type() != Type.YES || allocation.hasPendingAsyncFetch()) { - AllocationDecision allocationDecision = allocation.hasPendingAsyncFetch() - ? AllocationDecision.AWAITING_INFO - : AllocationDecision.fromDecisionType(canRebalance.type()); - return MoveDecision.cannotRebalance(canRebalance, allocationDecision, currentNodeWeightRanking, nodeDecisions); + // can not rebalance + return MoveDecision.rebalance( + canRemain, + canRebalance, + allocation.hasPendingAsyncFetch() + ? AllocationDecision.AWAITING_INFO + : AllocationDecision.fromDecisionType(canRebalance.type()), + null, + currentNodeWeightRanking, + nodeDecisions + ); } else { return MoveDecision.rebalance( + canRemain, canRebalance, AllocationDecision.fromDecisionType(rebalanceDecisionType), - assignedNode != null ? assignedNode.routingNode.node() : null, + targetNode != null ? targetNode.routingNode.node() : null, currentNodeWeightRanking, nodeDecisions ); @@ -888,7 +892,7 @@ public MoveDecision decideMove(final ShardRouting shardRouting) { RoutingNode routingNode = sourceNode.getRoutingNode(); Decision canRemain = allocation.deciders().canRemain(shardRouting, routingNode, allocation); if (canRemain.type() != Decision.Type.NO) { - return MoveDecision.stay(canRemain); + return MoveDecision.remain(canRemain); } sorter.reset(shardRouting.getIndexName()); @@ -917,16 +921,14 @@ private MoveDecision decideMove( final boolean explain = allocation.debugDecision(); Type bestDecision = Type.NO; RoutingNode targetNode = null; - final List nodeExplanationMap = explain ? new ArrayList<>() : null; + final List nodeResults = explain ? new ArrayList<>() : null; int weightRanking = 0; for (ModelNode currentNode : sorter.modelNodes) { if (currentNode != sourceNode) { RoutingNode target = currentNode.getRoutingNode(); Decision allocationDecision = decider.apply(shardRouting, target); if (explain) { - nodeExplanationMap.add( - new NodeAllocationResult(currentNode.getRoutingNode().node(), allocationDecision, ++weightRanking) - ); + nodeResults.add(new NodeAllocationResult(currentNode.getRoutingNode().node(), allocationDecision, ++weightRanking)); } // TODO maybe we can respect throttling here too? if (allocationDecision.type().higherThan(bestDecision)) { @@ -943,11 +945,11 @@ private MoveDecision decideMove( } } - return MoveDecision.cannotRemain( + return MoveDecision.move( remainDecision, AllocationDecision.fromDecisionType(bestDecision), targetNode != null ? targetNode.node() : null, - nodeExplanationMap + nodeResults ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java index 7f9720b64cca6..296acc30a83f5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/elasticsearch/cluster/service/MasterService.java @@ -524,6 +524,24 @@ public Builder incrementVersion(ClusterState clusterState) { return ClusterState.builder(clusterState).incrementVersion(); } + private static boolean versionNumbersPreserved(ClusterState oldState, ClusterState newState) { + if (oldState.nodes().getMasterNodeId() == null && newState.nodes().getMasterNodeId() != null) { + return true; // NodeJoinExecutor is special, we trust it to do the right thing with versions + } + + if (oldState.version() != newState.version()) { + return false; + } + if (oldState.metadata().version() != newState.metadata().version()) { + return false; + } + if (oldState.routingTable().version() != newState.routingTable().version()) { + // GatewayService is special and for odd legacy reasons gets to do this: + return oldState.clusterRecovered() == false && newState.clusterRecovered() && newState.routingTable().version() == 0; + } + return true; + } + /** * Submits an unbatched cluster state update task. This method exists for legacy reasons but is deprecated and forbidden in new * production code because unbatched tasks are a source of performance and stability bugs. You should instead implement your update @@ -1035,6 +1053,8 @@ private static boolean assertAllTasksComple return true; } + static final String TEST_ONLY_EXECUTOR_MAY_CHANGE_VERSION_NUMBER_TRANSIENT_NAME = "test_only_executor_may_change_version_number"; + private static ClusterState innerExecuteTasks( ClusterState previousClusterState, List> executionResults, @@ -1047,13 +1067,23 @@ private static ClusterState innerExecuteTas // to avoid leaking headers in production that were missed by tests try { - return executor.execute( + final var updatedState = executor.execute( new ClusterStateTaskExecutor.BatchExecutionContext<>( previousClusterState, executionResults, threadContext::newStoredContext ) ); + if (versionNumbersPreserved(previousClusterState, updatedState) == false) { + // Shenanigans! Executors mustn't meddle with version numbers. Perhaps the executor based its update on the wrong + // initial state, potentially losing an intervening cluster state update. That'd be very bad! + final var exception = new IllegalStateException( + "cluster state update executor did not preserve version numbers: [" + summary.toString() + "]" + ); + assert threadContext.getTransient(TEST_ONLY_EXECUTOR_MAY_CHANGE_VERSION_NUMBER_TRANSIENT_NAME) != null : exception; + throw exception; + } + return updatedState; } catch (Exception e) { logger.trace( () -> format( diff --git a/server/src/main/java/org/elasticsearch/common/bytes/BytesArray.java b/server/src/main/java/org/elasticsearch/common/bytes/BytesArray.java index 40697a0c158a5..3d26f2785a09e 100644 --- a/server/src/main/java/org/elasticsearch/common/bytes/BytesArray.java +++ b/server/src/main/java/org/elasticsearch/common/bytes/BytesArray.java @@ -59,16 +59,19 @@ public byte get(int index) { @Override public int indexOf(byte marker, int from) { final int len = length - from; - int off = offset + from; - final int toIndex = offset + length; + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final int offsetAsLocal = offset; + int off = offsetAsLocal + from; + final int toIndex = offsetAsLocal + length; + final byte[] bytesAsLocal = bytes; // First, try to find the marker in the first few bytes, so we can enter the faster 8-byte aligned loop below. // The idea for this logic is taken from Netty's io.netty.buffer.ByteBufUtil.firstIndexOf and optimized for little endian hardware. // See e.g. https://richardstartin.github.io/posts/finding-bytes for the idea behind this optimization. final int byteCount = len & 7; if (byteCount > 0) { - final int index = unrolledFirstIndexOf(bytes, off, byteCount, marker); + final int index = unrolledFirstIndexOf(bytesAsLocal, off, byteCount, marker); if (index != -1) { - return index - offset; + return index - offsetAsLocal; } off += byteCount; if (off == toIndex) { @@ -79,9 +82,9 @@ public int indexOf(byte marker, int from) { // faster SWAR (SIMD Within A Register) loop final long pattern = compilePattern(marker); for (int i = 0; i < longCount; i++) { - int index = findInLong(ByteUtils.readLongLE(bytes, off), pattern); + int index = findInLong(ByteUtils.readLongLE(bytesAsLocal, off), pattern); if (index < Long.BYTES) { - return off + index - offset; + return off + index - offsetAsLocal; } off += Long.BYTES; } diff --git a/server/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java b/server/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java index 22bed3ea0b1e9..42326566743ff 100644 --- a/server/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/bytes/BytesReferenceStreamInput.java @@ -56,6 +56,8 @@ public byte readByte() throws IOException { @Override public short readShort() throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer slice = this.slice; if (slice.remaining() >= 2) { return slice.getShort(); } else { @@ -66,6 +68,8 @@ public short readShort() throws IOException { @Override public int readInt() throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer slice = this.slice; if (slice.remaining() >= 4) { return slice.getInt(); } else { @@ -76,6 +80,8 @@ public int readInt() throws IOException { @Override public long readLong() throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer slice = this.slice; if (slice.remaining() >= 8) { return slice.getLong(); } else { @@ -87,6 +93,8 @@ public long readLong() throws IOException { @Override public String readString() throws IOException { final int chars = readArraySize(); + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer slice = this.slice; if (slice.hasArray()) { // attempt reading bytes directly into a string to minimize copying final String string = tryReadStringFromBytes( @@ -104,6 +112,8 @@ public String readString() throws IOException { @Override public int readVInt() throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer slice = this.slice; if (slice.remaining() >= 5) { return ByteBufferStreamInput.readVInt(slice); } @@ -112,6 +122,8 @@ public int readVInt() throws IOException { @Override public long readVLong() throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer slice = this.slice; if (slice.remaining() >= 10) { return ByteBufferStreamInput.readVLong(slice); } else { @@ -161,6 +173,8 @@ public int read() throws IOException { @Override public int read(final byte[] b, final int bOffset, final int len) throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer slice = this.slice; if (slice.remaining() >= len) { slice.get(b, bOffset, len); return len; @@ -226,6 +240,8 @@ private int skipMultiple(long n) throws IOException { int remaining = numBytesSkipped; while (remaining > 0) { maybeNextSlice(); + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer slice = this.slice; int currentLen = Math.min(remaining, slice.remaining()); remaining -= currentLen; slice.position(slice.position() + currentLen); diff --git a/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java b/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java index 65a3bf95336c6..9b8c06426e97c 100644 --- a/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java +++ b/server/src/main/java/org/elasticsearch/common/bytes/CompositeBytesReference.java @@ -116,17 +116,20 @@ public int indexOf(byte marker, int from) { } final int firstReferenceIndex = getOffsetIndex(from); - for (int i = firstReferenceIndex; i < references.length; ++i) { - final BytesReference reference = references[i]; + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final BytesReference[] referencesAsLocal = references; + final int[] offsetsAsLocal = offsets; + for (int i = firstReferenceIndex; i < referencesAsLocal.length; ++i) { + final BytesReference reference = referencesAsLocal[i]; final int internalFrom; if (i == firstReferenceIndex) { - internalFrom = from - offsets[firstReferenceIndex]; + internalFrom = from - offsetsAsLocal[firstReferenceIndex]; } else { internalFrom = 0; } result = reference.indexOf(marker, internalFrom); if (result != -1) { - result += offsets[i]; + result += offsetsAsLocal[i]; break; } } diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java index 41d129406551f..f1c0486a02d81 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java @@ -123,6 +123,8 @@ public static long readVLong(ByteBuffer buffer) throws IOException { @Override public String readString() throws IOException { final int chars = readArraySize(); + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer buffer = this.buffer; if (buffer.hasArray()) { // attempt reading bytes directly into a string to minimize copying final String string = tryReadStringFromBytes( @@ -140,6 +142,8 @@ public String readString() throws IOException { @Override public int read() throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer buffer = this.buffer; if (buffer.hasRemaining() == false) { return -1; } @@ -157,6 +161,8 @@ public byte readByte() throws IOException { @Override public int read(byte[] b, int off, int len) throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer buffer = this.buffer; if (buffer.hasRemaining() == false) { return -1; } @@ -168,6 +174,8 @@ public int read(byte[] b, int off, int len) throws IOException { @Override public long skip(long n) throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer buffer = this.buffer; int remaining = buffer.remaining(); if (n > remaining) { buffer.position(buffer.limit()); @@ -257,6 +265,8 @@ protected void ensureCanReadBytes(int length) throws EOFException { @Override public BytesReference readSlicedBytesReference() throws IOException { + // cache object fields (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + final ByteBuffer buffer = this.buffer; if (buffer.hasArray()) { int len = readVInt(); var res = new BytesArray(buffer.array(), buffer.arrayOffset() + buffer.position(), len); diff --git a/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java b/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java index b0dc6d98fe16b..2bcc9b48ff1b8 100644 --- a/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/AbstractHash.java @@ -32,7 +32,7 @@ public long id(long index) { } protected final long id(long index, long id) { - return ids.set(index, id + 1) - 1; + return ids.getAndSet(index, id + 1) - 1; } @Override diff --git a/server/src/main/java/org/elasticsearch/common/util/BigArrays.java b/server/src/main/java/org/elasticsearch/common/util/BigArrays.java index 1e8b0cc83eaa6..199eaa83a2da3 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigArrays.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigArrays.java @@ -118,11 +118,9 @@ public byte get(long index) { } @Override - public byte set(long index, byte value) { + public void set(long index, byte value) { assert indexIsInt(index); - final byte ret = array[(int) index]; array[(int) index] = value; - return ret; } @Override @@ -215,13 +213,19 @@ public int get(long index) { } @Override - public int set(long index, int value) { + public int getAndSet(long index, int value) { assert index >= 0 && index < size(); final int ret = (int) VH_PLATFORM_NATIVE_INT.get(array, (int) index << 2); VH_PLATFORM_NATIVE_INT.set(array, (int) index << 2, value); return ret; } + @Override + public void set(long index, int value) { + assert index >= 0 && index < size(); + VH_PLATFORM_NATIVE_INT.set(array, (int) index << 2, value); + } + @Override public int increment(long index, int inc) { assert index >= 0 && index < size(); @@ -272,13 +276,19 @@ public long get(long index) { } @Override - public long set(long index, long value) { + public long getAndSet(long index, long value) { assert index >= 0 && index < size(); final long ret = (long) VH_PLATFORM_NATIVE_LONG.get(array, (int) index << 3); VH_PLATFORM_NATIVE_LONG.set(array, (int) index << 3, value); return ret; } + @Override + public void set(long index, long value) { + assert index >= 0 && index < size(); + VH_PLATFORM_NATIVE_LONG.set(array, (int) index << 3, value); + } + @Override public long increment(long index, long inc) { assert index >= 0 && index < size(); @@ -336,11 +346,9 @@ public double get(long index) { } @Override - public double set(long index, double value) { + public void set(long index, double value) { assert index >= 0 && index < size(); - final double ret = (double) VH_PLATFORM_NATIVE_DOUBLE.get(array, (int) index << 3); VH_PLATFORM_NATIVE_DOUBLE.set(array, (int) index << 3, value); - return ret; } @Override @@ -400,11 +408,9 @@ public float get(long index) { } @Override - public float set(long index, float value) { + public void set(long index, float value) { assert index >= 0 && index < size(); - final float ret = (float) VH_PLATFORM_NATIVE_FLOAT.get(array, (int) index << 2); VH_PLATFORM_NATIVE_FLOAT.set(array, (int) index << 2, value); - return ret; } @Override diff --git a/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java b/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java index 61848769e661d..1e714f122d885 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigByteArray.java @@ -47,13 +47,11 @@ public byte get(long index) { } @Override - public byte set(long index, byte value) { + public void set(long index, byte value) { final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); final byte[] page = getPageForWriting(pageIndex); - final byte ret = page[indexInPage]; page[indexInPage] = value; - return ret; } @Override diff --git a/server/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java b/server/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java index 27dc454c85adf..3135ebb293070 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigDoubleArray.java @@ -42,13 +42,11 @@ public double get(long index) { } @Override - public double set(long index, double value) { + public void set(long index, double value) { final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); final byte[] page = getPageForWriting(pageIndex); - final double ret = (double) VH_PLATFORM_NATIVE_DOUBLE.get(page, indexInPage << 3); VH_PLATFORM_NATIVE_DOUBLE.set(page, indexInPage << 3, value); - return ret; } @Override diff --git a/server/src/main/java/org/elasticsearch/common/util/BigFloatArray.java b/server/src/main/java/org/elasticsearch/common/util/BigFloatArray.java index 9502950c1d25b..380b2c8e12b34 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigFloatArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigFloatArray.java @@ -30,13 +30,11 @@ final class BigFloatArray extends AbstractBigByteArray implements FloatArray { } @Override - public float set(long index, float value) { + public void set(long index, float value) { final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); final byte[] page = getPageForWriting(pageIndex); - final float ret = (float) VH_PLATFORM_NATIVE_FLOAT.get(page, indexInPage << 2); VH_PLATFORM_NATIVE_FLOAT.set(page, indexInPage << 2, value); - return ret; } @Override diff --git a/server/src/main/java/org/elasticsearch/common/util/BigIntArray.java b/server/src/main/java/org/elasticsearch/common/util/BigIntArray.java index 4388cc2308905..9ce9842c337c0 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigIntArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigIntArray.java @@ -46,7 +46,7 @@ public int get(long index) { } @Override - public int set(long index, int value) { + public int getAndSet(long index, int value) { final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); final byte[] page = getPageForWriting(pageIndex); @@ -55,6 +55,13 @@ public int set(long index, int value) { return ret; } + @Override + public void set(long index, int value) { + final int pageIndex = pageIndex(index); + final int indexInPage = indexInPage(index); + VH_PLATFORM_NATIVE_INT.set(getPageForWriting(pageIndex), indexInPage << 2, value); + } + @Override public int increment(long index, int inc) { final int pageIndex = pageIndex(index); diff --git a/server/src/main/java/org/elasticsearch/common/util/BigLongArray.java b/server/src/main/java/org/elasticsearch/common/util/BigLongArray.java index f0ccea26880c4..7d23e06f87658 100644 --- a/server/src/main/java/org/elasticsearch/common/util/BigLongArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/BigLongArray.java @@ -41,7 +41,7 @@ public long get(long index) { } @Override - public long set(long index, long value) { + public long getAndSet(long index, long value) { final int pageIndex = pageIndex(index); final int indexInPage = indexInPage(index); final byte[] page = getPageForWriting(pageIndex); @@ -50,6 +50,14 @@ public long set(long index, long value) { return ret; } + @Override + public void set(long index, long value) { + final int pageIndex = pageIndex(index); + final int indexInPage = indexInPage(index); + final byte[] page = getPageForWriting(pageIndex); + VH_PLATFORM_NATIVE_LONG.set(page, indexInPage << 3, value); + } + @Override public long increment(long index, long inc) { final int pageIndex = pageIndex(index); diff --git a/server/src/main/java/org/elasticsearch/common/util/ByteArray.java b/server/src/main/java/org/elasticsearch/common/util/ByteArray.java index cb2b10632d08b..2c16e730635f8 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ByteArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/ByteArray.java @@ -32,9 +32,9 @@ static ByteArray readFrom(StreamInput in) throws IOException { byte get(long index); /** - * Set a value at the given index and return the previous value. + * Set a value at the given index. */ - byte set(long index, byte value); + void set(long index, byte value); /** * Get a reference to a slice. diff --git a/server/src/main/java/org/elasticsearch/common/util/DoubleArray.java b/server/src/main/java/org/elasticsearch/common/util/DoubleArray.java index dde1157c905c7..80348d3b2945f 100644 --- a/server/src/main/java/org/elasticsearch/common/util/DoubleArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/DoubleArray.java @@ -28,9 +28,9 @@ static DoubleArray readFrom(StreamInput in) throws IOException { double get(long index); /** - * Set a value at the given index and return the previous value. + * Set a value at the given index. */ - double set(long index, double value); + void set(long index, double value); /** * Increment value at the given index by inc and return the value. diff --git a/server/src/main/java/org/elasticsearch/common/util/FloatArray.java b/server/src/main/java/org/elasticsearch/common/util/FloatArray.java index 33427299fe26c..057f51f45b1f6 100644 --- a/server/src/main/java/org/elasticsearch/common/util/FloatArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/FloatArray.java @@ -19,9 +19,9 @@ public interface FloatArray extends BigArray { float get(long index); /** - * Set a value at the given index and return the previous value. + * Set a value at the given index. */ - float set(long index, float value); + void set(long index, float value); /** * Fill slots between fromIndex inclusive to toIndex exclusive with value. diff --git a/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java b/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java index 051dd31ce8869..d9beea76b371a 100644 --- a/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java +++ b/server/src/main/java/org/elasticsearch/common/util/Int3Hash.java @@ -132,9 +132,9 @@ protected void removeAndAdd(long index) { final long id = id(index, -1); assert id >= 0; long keyOffset = id * 3; - final int key1 = keys.set(keyOffset, 0); - final int key2 = keys.set(keyOffset + 1, 0); - final int key3 = keys.set(keyOffset + 2, 0); + final int key1 = keys.getAndSet(keyOffset, 0); + final int key2 = keys.getAndSet(keyOffset + 1, 0); + final int key3 = keys.getAndSet(keyOffset + 2, 0); reset(key1, key2, key3, id); } diff --git a/server/src/main/java/org/elasticsearch/common/util/IntArray.java b/server/src/main/java/org/elasticsearch/common/util/IntArray.java index 06975ffba46da..4f4dd61863595 100644 --- a/server/src/main/java/org/elasticsearch/common/util/IntArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/IntArray.java @@ -29,7 +29,12 @@ static IntArray readFrom(StreamInput in) throws IOException { /** * Set a value at the given index and return the previous value. */ - int set(long index, int value); + int getAndSet(long index, int value); + + /** + * Set a value at the given index + */ + void set(long index, int value); /** * Increment value at the given index by inc and return the value. diff --git a/server/src/main/java/org/elasticsearch/common/util/LongArray.java b/server/src/main/java/org/elasticsearch/common/util/LongArray.java index 59321d1957f4d..cff8c86eef4b6 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongArray.java @@ -30,7 +30,12 @@ static LongArray readFrom(StreamInput in) throws IOException { /** * Set a value at the given index and return the previous value. */ - long set(long index, long value); + long getAndSet(long index, long value); + + /** + * Set a value at the given index. + */ + void set(long index, long value); /** * Increment value at the given index by inc and return the value. diff --git a/server/src/main/java/org/elasticsearch/common/util/LongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongHash.java index 6ca4d9f0986f6..32364f8d2f341 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongHash.java @@ -110,7 +110,7 @@ public long add(long key) { protected void removeAndAdd(long index) { final long id = id(index, -1); assert id >= 0; - final long key = keys.set(id, 0); + final long key = keys.getAndSet(id, 0); reset(key, id); } diff --git a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java index 13405d491298c..61dd3b457029c 100644 --- a/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java +++ b/server/src/main/java/org/elasticsearch/common/util/LongLongHash.java @@ -134,8 +134,8 @@ protected void removeAndAdd(long index) { final long id = id(index, -1); assert id >= 0; long keyOffset = id * 2; - final long key1 = keys.set(keyOffset, 0); - final long key2 = keys.set(keyOffset + 1, 0); + final long key1 = keys.getAndSet(keyOffset, 0); + final long key2 = keys.getAndSet(keyOffset + 1, 0); reset(key1, key2, id); } diff --git a/server/src/main/java/org/elasticsearch/common/util/ReleasableByteArray.java b/server/src/main/java/org/elasticsearch/common/util/ReleasableByteArray.java index ce0f5bdfedd40..0eea3443391c1 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ReleasableByteArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/ReleasableByteArray.java @@ -62,7 +62,7 @@ public boolean get(long index, int len, BytesRef ref) { } @Override - public byte set(long index, byte value) { + public void set(long index, byte value) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/common/util/ReleasableDoubleArray.java b/server/src/main/java/org/elasticsearch/common/util/ReleasableDoubleArray.java index 61b2f52ee384e..d7279b845f225 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ReleasableDoubleArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/ReleasableDoubleArray.java @@ -44,7 +44,7 @@ public double get(long index) { } @Override - public double set(long index, double value) { + public void set(long index, double value) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/common/util/ReleasableIntArray.java b/server/src/main/java/org/elasticsearch/common/util/ReleasableIntArray.java index 2b433f6812a87..9dbc11328974a 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ReleasableIntArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/ReleasableIntArray.java @@ -44,7 +44,12 @@ public int get(long index) { } @Override - public int set(long index, int value) { + public int getAndSet(long index, int value) { + throw new UnsupportedOperationException(); + } + + @Override + public void set(long index, int value) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/common/util/ReleasableLongArray.java b/server/src/main/java/org/elasticsearch/common/util/ReleasableLongArray.java index 2980713e2e652..4f36cdc890d78 100644 --- a/server/src/main/java/org/elasticsearch/common/util/ReleasableLongArray.java +++ b/server/src/main/java/org/elasticsearch/common/util/ReleasableLongArray.java @@ -45,7 +45,12 @@ public long get(long index) { } @Override - public long set(long index, long value) { + public long getAndSet(long index, long value) { + throw new UnsupportedOperationException(); + } + + @Override + public void set(long index, long value) { throw new UnsupportedOperationException(); } diff --git a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java index 04a71b2421ddc..1303bdbfde1eb 100644 --- a/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java +++ b/server/src/main/java/org/elasticsearch/common/util/concurrent/EsExecutors.java @@ -103,22 +103,63 @@ public static EsThreadPoolExecutor newScaling( TimeUnit unit, boolean rejectAfterShutdown, ThreadFactory threadFactory, - ThreadContext contextHolder + ThreadContext contextHolder, + TaskTrackingConfig config ) { ExecutorScalingQueue queue = new ExecutorScalingQueue<>(); - EsThreadPoolExecutor executor = new EsThreadPoolExecutor( + EsThreadPoolExecutor executor; + if (config.trackExecutionTime()) { + executor = new TaskExecutionTimeTrackingEsThreadPoolExecutor( + name, + min, + max, + keepAliveTime, + unit, + queue, + TimedRunnable::new, + threadFactory, + new ForceQueuePolicy(rejectAfterShutdown), + contextHolder, + config + ); + } else { + executor = new EsThreadPoolExecutor( + name, + min, + max, + keepAliveTime, + unit, + queue, + threadFactory, + new ForceQueuePolicy(rejectAfterShutdown), + contextHolder + ); + } + queue.executor = executor; + return executor; + } + + public static EsThreadPoolExecutor newScaling( + String name, + int min, + int max, + long keepAliveTime, + TimeUnit unit, + boolean rejectAfterShutdown, + ThreadFactory threadFactory, + ThreadContext contextHolder + ) { + return newScaling( name, min, max, keepAliveTime, unit, - queue, + rejectAfterShutdown, threadFactory, - new ForceQueuePolicy(rejectAfterShutdown), - contextHolder + contextHolder, + TaskTrackingConfig.DO_NOT_TRACK ); - queue.executor = executor; - return executor; } public static EsThreadPoolExecutor newFixed( diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java index 31d996500cd83..599517b481eeb 100644 --- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java +++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayStartedShards.java @@ -205,11 +205,6 @@ public ShardId shardId() { public String getCustomDataPath() { return customDataPath; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } } public static class NodesGatewayStartedShards extends BaseNodesResponse { diff --git a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsAction.java b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsAction.java index 9833a5368f058..394e14a60d26e 100644 --- a/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsAction.java +++ b/server/src/main/java/org/elasticsearch/health/stats/HealthApiStatsAction.java @@ -44,11 +44,6 @@ public Request() { super((String[]) null); } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - @Override public String toString() { return "health_api_stats"; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 74c2c57594e72..f190462d6d1e9 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -154,7 +154,7 @@ public IndexSortConfig(IndexSettings indexSettings) { List fields = INDEX_SORT_FIELD_SETTING.get(settings); if (this.indexMode == IndexMode.LOGS && fields.isEmpty()) { - fields = List.of("hostname", DataStream.TIMESTAMP_FIELD_NAME); + fields = List.of("host.name", DataStream.TIMESTAMP_FIELD_NAME); } this.sortSpecs = fields.stream().map(FieldSortSpec::new).toArray(FieldSortSpec[]::new); diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java index e04a829acd461..701bf5dc98552 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES813Int8FlatVectorFormat.java @@ -37,16 +37,15 @@ public class ES813Int8FlatVectorFormat extends KnnVectorsFormat { private final FlatVectorsFormat format; public ES813Int8FlatVectorFormat() { - this(null); + this(null, 7, false); } /** * Sole constructor */ - public ES813Int8FlatVectorFormat(Float confidenceInterval) { + public ES813Int8FlatVectorFormat(Float confidenceInterval, int bits, boolean compress) { super(NAME); - int bits = 7; - boolean compress = false; + // TODO can we just switch this to ES814ScalarQuantizedVectorsFormat ? this.format = new Lucene99ScalarQuantizedVectorsFormat(confidenceInterval, bits, compress); } @@ -60,6 +59,11 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException return new ES813FlatVectorReader(format.fieldsReader(state)); } + @Override + public String toString() { + return NAME + "(name=" + NAME + ", innerFormat=" + format + ")"; + } + public static class ES813FlatVectorWriter extends KnnVectorsWriter { private final FlatVectorsWriter writer; diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormat.java index 22baa026a7e7c..24c9a67965735 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814HnswScalarQuantizedVectorsFormat.java @@ -16,14 +16,11 @@ import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.TaskExecutor; import java.io.IOException; -import java.util.concurrent.ExecutorService; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; public final class ES814HnswScalarQuantizedVectorsFormat extends KnnVectorsFormat { @@ -39,20 +36,11 @@ public final class ES814HnswScalarQuantizedVectorsFormat extends KnnVectorsForma /** The format for storing, reading, merging vectors on disk */ private final FlatVectorsFormat flatVectorsFormat; - private final int numMergeWorkers; - private final TaskExecutor mergeExec; - public ES814HnswScalarQuantizedVectorsFormat() { - this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, DEFAULT_NUM_MERGE_WORKER, null, null); + this(DEFAULT_MAX_CONN, DEFAULT_BEAM_WIDTH, null, 7, false); } - public ES814HnswScalarQuantizedVectorsFormat( - int maxConn, - int beamWidth, - int numMergeWorkers, - Float confidenceInterval, - ExecutorService mergeExec - ) { + public ES814HnswScalarQuantizedVectorsFormat(int maxConn, int beamWidth, Float confidenceInterval, int bits, boolean compress) { super(NAME); if (maxConn <= 0 || maxConn > MAXIMUM_MAX_CONN) { throw new IllegalArgumentException( @@ -66,24 +54,12 @@ public ES814HnswScalarQuantizedVectorsFormat( } this.maxConn = maxConn; this.beamWidth = beamWidth; - if (numMergeWorkers > 1 && mergeExec == null) { - throw new IllegalArgumentException("No executor service passed in when " + numMergeWorkers + " merge workers are requested"); - } - if (numMergeWorkers == 1 && mergeExec != null) { - throw new IllegalArgumentException("No executor service is needed as we'll use single thread to merge"); - } - this.numMergeWorkers = numMergeWorkers; - if (mergeExec != null) { - this.mergeExec = new TaskExecutor(mergeExec); - } else { - this.mergeExec = null; - } - this.flatVectorsFormat = new ES814ScalarQuantizedVectorsFormat(confidenceInterval); + this.flatVectorsFormat = new ES814ScalarQuantizedVectorsFormat(confidenceInterval, bits, compress); } @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), numMergeWorkers, mergeExec); + return new Lucene99HnswVectorsWriter(state, maxConn, beamWidth, flatVectorsFormat.fieldsWriter(state), 1, null); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java index 0d1c5efeb3e28..c4b52d26fc6e7 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java +++ b/server/src/main/java/org/elasticsearch/index/codec/vectors/ES814ScalarQuantizedVectorsFormat.java @@ -35,14 +35,17 @@ import org.apache.lucene.util.quantization.QuantizedVectorsReader; import org.apache.lucene.util.quantization.RandomAccessQuantizedByteVectorValues; import org.apache.lucene.util.quantization.ScalarQuantizer; -import org.elasticsearch.vec.VectorScorerFactory; -import org.elasticsearch.vec.VectorSimilarityType; +import org.elasticsearch.simdvec.VectorScorerFactory; +import org.elasticsearch.simdvec.VectorSimilarityType; import java.io.IOException; +import static org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat.DYNAMIC_CONFIDENCE_INTERVAL; + public class ES814ScalarQuantizedVectorsFormat extends FlatVectorsFormat { static final String NAME = "ES814ScalarQuantizedVectorsFormat"; + private static final int ALLOWED_BITS = (1 << 8) | (1 << 7) | (1 << 4); private static final FlatVectorsFormat rawVectorFormat = new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); @@ -59,8 +62,12 @@ public class ES814ScalarQuantizedVectorsFormat extends FlatVectorsFormat { public final Float confidenceInterval; final FlatVectorsScorer flatVectorScorer; - public ES814ScalarQuantizedVectorsFormat(Float confidenceInterval) { + private final byte bits; + private final boolean compress; + + public ES814ScalarQuantizedVectorsFormat(Float confidenceInterval, int bits, boolean compress) { if (confidenceInterval != null + && confidenceInterval != DYNAMIC_CONFIDENCE_INTERVAL && (confidenceInterval < MINIMUM_CONFIDENCE_INTERVAL || confidenceInterval > MAXIMUM_CONFIDENCE_INTERVAL)) { throw new IllegalArgumentException( "confidenceInterval must be between " @@ -71,19 +78,42 @@ public ES814ScalarQuantizedVectorsFormat(Float confidenceInterval) { + confidenceInterval ); } + if (bits < 1 || bits > 8 || (ALLOWED_BITS & (1 << bits)) == 0) { + throw new IllegalArgumentException("bits must be one of: 4, 7, 8; bits=" + bits); + } this.confidenceInterval = confidenceInterval; this.flatVectorScorer = new ESFlatVectorsScorer(new ScalarQuantizedVectorScorer(DefaultFlatVectorScorer.INSTANCE)); + this.bits = (byte) bits; + this.compress = compress; } @Override public String toString() { - return NAME + "(name=" + NAME + ", confidenceInterval=" + confidenceInterval + ", rawVectorFormat=" + rawVectorFormat + ")"; + return NAME + + "(name=" + + NAME + + ", confidenceInterval=" + + confidenceInterval + + ", bits=" + + bits + + ", compressed=" + + compress + + ", rawVectorFormat=" + + rawVectorFormat + + ")"; } @Override public FlatVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { return new ES814ScalarQuantizedVectorsWriter( - new Lucene99ScalarQuantizedVectorsWriter(state, confidenceInterval, rawVectorFormat.fieldsWriter(state), flatVectorScorer) + new Lucene99ScalarQuantizedVectorsWriter( + state, + confidenceInterval, + bits, + compress, + rawVectorFormat.fieldsWriter(state), + flatVectorScorer + ) ); } @@ -208,6 +238,10 @@ static final class ESFlatVectorsScorer implements FlatVectorsScorer { public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarityFunction sim, RandomAccessVectorValues values) throws IOException { if (values instanceof RandomAccessQuantizedByteVectorValues qValues && values.getSlice() != null) { + // TODO: optimize int4 quantization + if (qValues.getScalarQuantizer().getBits() != 7) { + return delegate.getRandomVectorScorerSupplier(sim, values); + } if (factory != null) { var scorer = factory.getInt7SQVectorScorerSupplier( VectorSimilarityType.of(sim), @@ -227,6 +261,10 @@ public RandomVectorScorerSupplier getRandomVectorScorerSupplier(VectorSimilarity public RandomVectorScorer getRandomVectorScorer(VectorSimilarityFunction sim, RandomAccessVectorValues values, float[] query) throws IOException { if (values instanceof RandomAccessQuantizedByteVectorValues qValues && values.getSlice() != null) { + // TODO: optimize int4 quantization + if (qValues.getScalarQuantizer().getBits() != 7) { + return delegate.getRandomVectorScorer(sim, values, query); + } if (factory != null) { var scorer = factory.getInt7SQVectorScorer(sim, qValues, query); if (scorer.isPresent()) { diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 4f461a5d51c75..1d62debd77e7f 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -23,6 +23,8 @@ import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.QueryCachingPolicy; @@ -59,11 +61,15 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.mapper.DocumentParser; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.seqno.SequenceNumbers; @@ -71,6 +77,7 @@ import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardLongFieldRange; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.index.translog.TranslogStats; @@ -273,6 +280,61 @@ private long getDenseVectorValueCount(final LeafReader atomicReader) throws IOEx return count; } + /** + * Returns the {@link SparseVectorStats} for this engine + */ + public SparseVectorStats sparseVectorStats(MappingLookup mappingLookup) { + try (Searcher searcher = acquireSearcher(DOC_STATS_SOURCE, SearcherScope.INTERNAL)) { + return sparseVectorStats(searcher.getIndexReader(), mappingLookup); + } + } + + protected final SparseVectorStats sparseVectorStats(IndexReader indexReader, MappingLookup mappingLookup) { + long valueCount = 0; + + if (mappingLookup == null) { + return new SparseVectorStats(valueCount); + } + + // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause + // the next scheduled refresh to go through and refresh the stats as well + for (LeafReaderContext readerContext : indexReader.leaves()) { + try { + valueCount += getSparseVectorValueCount(readerContext.reader(), mappingLookup); + } catch (IOException e) { + logger.trace(() -> "failed to get sparse vector stats for [" + readerContext + "]", e); + } + } + return new SparseVectorStats(valueCount); + } + + private long getSparseVectorValueCount(final LeafReader atomicReader, MappingLookup mappingLookup) throws IOException { + long count = 0; + + Map mappers = new HashMap<>(); + for (Mapper mapper : mappingLookup.fieldMappers()) { + if (mapper instanceof FieldMapper fieldMapper) { + if (fieldMapper.fieldType() instanceof SparseVectorFieldMapper.SparseVectorFieldType) { + mappers.put(fieldMapper.name(), fieldMapper); + } + } + } + + for (FieldInfo info : atomicReader.getFieldInfos()) { + String name = info.name; + if (mappers.containsKey(name)) { + Terms terms = atomicReader.terms(FieldNamesFieldMapper.NAME); + if (terms != null) { + TermsEnum termsEnum = terms.iterator(); + if (termsEnum.seekExact(new BytesRef(name))) { + count += termsEnum.docFreq(); + } + } + } + } + return count; + } + /** * Performs the pre-closing checks on the {@link Engine}. * diff --git a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java index 7a817500c4ca5..51840d2fbfcdd 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java +++ b/server/src/main/java/org/elasticsearch/index/engine/EngineConfig.java @@ -24,6 +24,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.CodecService; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.seqno.RetentionLeases; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.store.Store; @@ -51,6 +52,7 @@ public final class EngineConfig { private volatile boolean enableGcDeletes = true; private final TimeValue flushMergesAfter; private final String codecName; + private final MapperService mapperService; private final IndexStorePlugin.SnapshotCommitSupplier snapshotCommitSupplier; private final ThreadPool threadPool; private final Engine.Warmer warmer; @@ -163,7 +165,8 @@ public EngineConfig( Comparator leafSorter, LongSupplier relativeTimeInNanosSupplier, Engine.IndexCommitListener indexCommitListener, - boolean promotableToPrimary + boolean promotableToPrimary, + MapperService mapperService ) { this.shardId = shardId; this.indexSettings = indexSettings; @@ -176,6 +179,7 @@ public EngineConfig( this.codecService = codecService; this.eventListener = eventListener; codecName = indexSettings.getValue(INDEX_CODEC_SETTING); + this.mapperService = mapperService; // We need to make the indexing buffer for this shard at least as large // as the amount of memory that is available for all engines on the // local node so that decisions to flush segments to disk are made by @@ -436,4 +440,8 @@ public boolean isPromotableToPrimary() { public boolean getUseCompoundFile() { return useCompoundFile; } + + public MapperService getMapperService() { + return mapperService; + } } diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index 245cef2d97b24..be64365fedd34 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -2223,6 +2223,14 @@ protected void flushHoldingLock(boolean force, boolean waitIfOngoing, ActionList // we need to refresh in order to clear older version values refresh("version_table_flush", SearcherScope.INTERNAL, true); translog.trimUnreferencedReaders(); + // Update the translog location for flushListener if (1) the writeLocation has changed during the flush and + // (2) indexWriter has committed all the changes (checks must be done in this order). + // If the indexWriter has uncommitted changes, they will be flushed by the next flush as intended. + final Translog.Location writeLocationAfterFlush = translog.getLastWriteLocation(); + if (writeLocationAfterFlush.equals(commitLocation) == false && hasUncommittedChanges() == false) { + assert writeLocationAfterFlush.compareTo(commitLocation) > 0 : writeLocationAfterFlush + " <= " + commitLocation; + commitLocation = writeLocationAfterFlush; + } // Use the timestamp from when the flush started, but only update it in case of success, so that any exception in // the above lines would not lead the engine to think that it recently flushed, when it did not. this.lastFlushTimestamp = lastFlushTimestamp; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 78e0c14b81e20..f6669075480dd 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -95,6 +95,12 @@ protected void throwIfEmpty() { } } + protected void throwIfBeyondLength(int i) { + if (i >= size()) { + throw new IndexOutOfBoundsException("A document doesn't have a value for a field at position [" + i + "]!"); + } + } + public static class Longs extends ScriptDocValues { public Longs(Supplier supplier) { @@ -108,6 +114,7 @@ public long getValue() { @Override public Long get(int index) { throwIfEmpty(); + throwIfBeyondLength(index); return supplier.getInternal(index); } @@ -133,12 +140,7 @@ public ZonedDateTime getValue() { @Override public ZonedDateTime get(int index) { - if (supplier.size() == 0) { - throw new IllegalStateException( - "A document doesn't have a value for a field! " - + "Use doc[].size()==0 to check if a document is missing a field!" - ); - } + throwIfEmpty(); if (index >= supplier.size()) { throw new IndexOutOfBoundsException( "attempted to fetch the [" + index + "] date when there are only [" + supplier.size() + "] dates." @@ -207,12 +209,8 @@ public double getValue() { @Override public Double get(int index) { - if (supplier.size() == 0) { - throw new IllegalStateException( - "A document doesn't have a value for a field! " - + "Use doc[].size()==0 to check if a document is missing a field!" - ); - } + throwIfEmpty(); + throwIfBeyondLength(index); return supplier.getInternal(index); } @@ -312,12 +310,8 @@ public double getLon() { @Override public GeoPoint get(int index) { - if (supplier.size() == 0) { - throw new IllegalStateException( - "A document doesn't have a value for a field! " - + "Use doc[].size()==0 to check if a document is missing a field!" - ); - } + throwIfEmpty(); + throwIfBeyondLength(index); final GeoPoint point = supplier.getInternal(index); return new GeoPoint(point.lat(), point.lon()); } @@ -408,6 +402,7 @@ public boolean getValue() { @Override public Boolean get(int index) { throwIfEmpty(); + throwIfBeyondLength(index); return supplier.getInternal(index); } @@ -484,12 +479,8 @@ public String getValue() { @Override public String get(int index) { - if (supplier.size() == 0) { - throw new IllegalStateException( - "A document doesn't have a value for a field! " - + "Use doc[].size()==0 to check if a document is missing a field!" - ); - } + throwIfEmpty(); + throwIfBeyondLength(index); return supplier.getInternal(index); } @@ -513,6 +504,7 @@ public BytesRef getValue() { @Override public BytesRef get(int index) { throwIfEmpty(); + throwIfBeyondLength(index); return supplier.getInternal(index); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 7cadec68f3e61..831244a3969ef 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -8,12 +8,14 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.search.Query; +import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeometryFormatterFactory; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.DeprecationHandler; import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.support.MapXContentParser; @@ -52,12 +54,12 @@ public abstract static class Parser { * Parse the given xContent value to one or more objects of type {@link T}. The value can be * in any supported format. */ - public abstract void parse(XContentParser parser, CheckedConsumer consumer, Consumer onMalformed) + public abstract void parse(XContentParser parser, CheckedConsumer consumer, MalformedValueHandler malformedHandler) throws IOException; private void fetchFromSource(Object sourceMap, Consumer consumer) { try (XContentParser parser = wrapObject(sourceMap)) { - parse(parser, v -> consumer.accept(normalizeFromSource(v)), e -> {}); /* ignore malformed */ + parse(parser, v -> consumer.accept(normalizeFromSource(v)), NoopMalformedValueHandler.INSTANCE); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -84,6 +86,36 @@ private static XContentParser wrapObject(Object sourceMap) throws IOException { } } + public interface MalformedValueHandler { + void notify(Exception parsingException) throws IOException; + + void notify(Exception parsingException, XContentBuilder malformedDataForSyntheticSource) throws IOException; + } + + public record NoopMalformedValueHandler() implements MalformedValueHandler { + public static final NoopMalformedValueHandler INSTANCE = new NoopMalformedValueHandler(); + + @Override + public void notify(Exception parsingException) {} + + @Override + public void notify(Exception parsingException, XContentBuilder malformedDataForSyntheticSource) {} + } + + public record DefaultMalformedValueHandler(CheckedBiConsumer consumer) + implements + MalformedValueHandler { + @Override + public void notify(Exception parsingException) throws IOException { + consumer.accept(parsingException, null); + } + + @Override + public void notify(Exception parsingException, XContentBuilder malformedDataForSyntheticSource) throws IOException { + consumer.accept(parsingException, malformedDataForSyntheticSource); + } + } + public abstract static class AbstractGeometryFieldType extends MappedFieldType { protected final Parser geometryParser; @@ -220,17 +252,20 @@ public final void parse(DocumentParserContext context) throws IOException { new IllegalArgumentException("Cannot index data directly into a field with a [script] parameter") ); } - parser.parse(context.parser(), v -> index(context, v), e -> { - if (ignoreMalformed()) { - context.addIgnoredField(fieldType().name()); - } else { - throw new DocumentParsingException( - context.parser().getTokenLocation(), - "failed to parse field [" + fieldType().name() + "] of type [" + contentType() + "]", - e - ); - } - }); + parser.parse(context.parser(), v -> index(context, v), new DefaultMalformedValueHandler((e, b) -> onMalformedValue(context, b, e))); + } + + protected void onMalformedValue(DocumentParserContext context, XContentBuilder malformedDataForSyntheticSource, Exception cause) + throws IOException { + if (ignoreMalformed()) { + context.addIgnoredField(fieldType().name()); + } else { + throw new DocumentParsingException( + context.parser().getTokenLocation(), + "failed to parse field [" + fieldType().name() + "] of type [" + contentType() + "]", + cause + ); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java index 9136a0dfbf550..2b4ecc8f0a89d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; -import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -71,7 +70,7 @@ public T getNullValue() { /** A base parser implementation for point formats */ protected abstract static class PointParser extends Parser { protected final String field; - private final CheckedFunction objectParser; + protected final CheckedFunction objectParser; private final T nullValue; private final boolean ignoreZValue; protected final boolean ignoreMalformed; @@ -98,7 +97,7 @@ protected PointParser( protected abstract T createPoint(double x, double y); @Override - public void parse(XContentParser parser, CheckedConsumer consumer, Consumer onMalformed) + public void parse(XContentParser parser, CheckedConsumer consumer, MalformedValueHandler malformedHandler) throws IOException { if (parser.currentToken() == XContentParser.Token.START_ARRAY) { XContentParser.Token token = parser.nextToken(); @@ -132,7 +131,7 @@ public void parse(XContentParser parser, CheckedConsumer consume consumer.accept(nullValue); } } else { - parseAndConsumeFromObject(parser, consumer, onMalformed); + parseAndConsumeFromObject(parser, consumer, malformedHandler); } token = parser.nextToken(); } @@ -142,20 +141,20 @@ public void parse(XContentParser parser, CheckedConsumer consume consumer.accept(nullValue); } } else { - parseAndConsumeFromObject(parser, consumer, onMalformed); + parseAndConsumeFromObject(parser, consumer, malformedHandler); } } - private void parseAndConsumeFromObject( + protected void parseAndConsumeFromObject( XContentParser parser, CheckedConsumer consumer, - Consumer onMalformed - ) { + MalformedValueHandler malformedHandler + ) throws IOException { try { T point = objectParser.apply(parser); consumer.accept(validate(point)); } catch (Exception e) { - onMalformed.accept(e); + malformedHandler.notify(e); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java index fefc49e470d58..42feda3e9dd48 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockLoader.java @@ -92,6 +92,16 @@ interface StoredFields { */ SortedSetDocValues ordinals(LeafReaderContext context) throws IOException; + /** + * In support of 'Union Types', we sometimes desire that Blocks loaded from source are immediately + * converted in some way. Typically, this would be a type conversion, or an encoding conversion. + * @param block original block loaded from source + * @return converted block (or original if no conversion required) + */ + default Block convert(Block block) { + return block; + } + /** * Load blocks with only null. */ @@ -456,6 +466,13 @@ interface BytesRefBuilder extends Builder { BytesRefBuilder appendBytesRef(BytesRef value); } + interface FloatBuilder extends Builder { + /** + * Appends a float to the current entry. + */ + FloatBuilder appendFloat(float value); + } + interface DoubleBuilder extends Builder { /** * Appends a double to the current entry. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java new file mode 100644 index 0000000000000..efc3c7b507300 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompositeSyntheticFieldLoader.java @@ -0,0 +1,174 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Collections.emptyList; + +/** + * A {@link SourceLoader.SyntheticFieldLoader} that uses a set of sub-loaders + * to produce synthetic source for the field. + * Typical use case is to gather field values from doc_values and append malformed values + * stored in a different field in case of ignore_malformed being enabled. + */ +public class CompositeSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader { + private final String fieldName; + private final String fullFieldName; + private final SyntheticFieldLoaderLayer[] parts; + private boolean hasValue; + + public CompositeSyntheticFieldLoader(String fieldName, String fullFieldName, SyntheticFieldLoaderLayer... parts) { + this.fieldName = fieldName; + this.fullFieldName = fullFieldName; + this.parts = parts; + this.hasValue = false; + } + + @Override + public Stream> storedFieldLoaders() { + return Arrays.stream(parts).flatMap(SyntheticFieldLoaderLayer::storedFieldLoaders).map(e -> Map.entry(e.getKey(), values -> { + hasValue = true; + e.getValue().load(values); + })); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + var loaders = new ArrayList(parts.length); + for (var part : parts) { + var partLoader = part.docValuesLoader(leafReader, docIdsInLeaf); + if (partLoader != null) { + loaders.add(partLoader); + } + } + + if (loaders.isEmpty()) { + return null; + } + + return docId -> { + boolean hasDocs = false; + for (var loader : loaders) { + hasDocs |= loader.advanceToDoc(docId); + } + + this.hasValue |= hasDocs; + return hasDocs; + }; + } + + @Override + public boolean hasValue() { + return hasValue; + } + + @Override + public void write(XContentBuilder b) throws IOException { + var totalCount = Arrays.stream(parts).mapToLong(SyntheticFieldLoaderLayer::valueCount).sum(); + + if (totalCount == 0) { + return; + } + + if (totalCount == 1) { + b.field(fieldName); + for (var part : parts) { + part.write(b); + } + return; + } + + b.startArray(fieldName); + for (var part : parts) { + part.write(b); + } + b.endArray(); + } + + @Override + public String fieldName() { + return this.fullFieldName; + } + + /** + * Represents one layer of loading synthetic source values for a field + * as a part of {@link CompositeSyntheticFieldLoader}. + *
    + * Note that the contract of {@link SourceLoader.SyntheticFieldLoader#write(XContentBuilder)} + * is slightly different here since it only needs to write field values without encompassing object or array. + */ + public interface SyntheticFieldLoaderLayer extends SourceLoader.SyntheticFieldLoader { + /** + * Number of values that this loader will write. + * @return + */ + long valueCount(); + } + + /** + * Layer that loads malformed values stored in a dedicated field with a conventional name. + * @see IgnoreMalformedStoredValues + */ + public static class MalformedValuesLayer implements SyntheticFieldLoaderLayer { + private final String fieldName; + private List values; + + public MalformedValuesLayer(String fieldName) { + this.fieldName = IgnoreMalformedStoredValues.name(fieldName); + this.values = emptyList(); + } + + @Override + public long valueCount() { + return values.size(); + } + + @Override + public Stream> storedFieldLoaders() { + return Stream.of(Map.entry(fieldName, values -> this.values = values)); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + return null; + } + + @Override + public boolean hasValue() { + return values.isEmpty() == false; + } + + @Override + public void write(XContentBuilder b) throws IOException { + for (Object v : values) { + if (v instanceof BytesRef r) { + XContentDataHelper.decodeAndWrite(b, r); + } else { + b.value(v); + } + } + values = emptyList(); + } + + @Override + public String fieldName() { + return fieldName; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 3d4f0823bb1cf..034e8fd0770f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -132,7 +132,8 @@ private static void internalParseDocument(MetadataFieldMapper[] metadataFieldsMa new IgnoredSourceFieldMapper.NameValue( MapperService.SINGLE_MAPPING_NAME, 0, - XContentDataHelper.encodeToken(context.parser()) + XContentDataHelper.encodeToken(context.parser()), + context.doc() ) ); } else { @@ -268,7 +269,8 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio new IgnoredSourceFieldMapper.NameValue( context.parent().fullPath(), context.parent().fullPath().indexOf(currentFieldName), - XContentDataHelper.encodeToken(parser) + XContentDataHelper.encodeToken(parser), + context.doc() ) ); } else { @@ -288,13 +290,14 @@ static void parseObjectOrNested(DocumentParserContext context) throws IOExceptio if (context.parent().isNested()) { // Handle a nested object that doesn't contain an array. Arrays are handled in #parseNonDynamicArray. - if (context.mappingLookup().isSourceSynthetic() && context.getClonedSource() == false) { + if (context.parent().storeArraySource() && context.mappingLookup().isSourceSynthetic() && context.getClonedSource() == false) { Tuple tuple = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( new IgnoredSourceFieldMapper.NameValue( context.parent().name(), context.parent().fullPath().indexOf(context.parent().simpleName()), - XContentDataHelper.encodeXContentBuilder(tuple.v2()) + XContentDataHelper.encodeXContentBuilder(tuple.v2()), + context.doc() ) ); context = tuple.v1(); @@ -661,9 +664,8 @@ private static void parseNonDynamicArray( && (objectMapper.storeArraySource() || objectMapper.dynamic == ObjectMapper.Dynamic.RUNTIME); boolean fieldWithFallbackSyntheticSource = mapper instanceof FieldMapper fieldMapper && fieldMapper.syntheticSourceMode() == FieldMapper.SyntheticSourceMode.FALLBACK; - boolean nestedObject = mapper instanceof NestedObjectMapper; boolean dynamicRuntimeContext = context.dynamic() == ObjectMapper.Dynamic.RUNTIME; - if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || nestedObject || dynamicRuntimeContext) { + if (objectRequiresStoringSource || fieldWithFallbackSyntheticSource || dynamicRuntimeContext) { Tuple tuple = XContentDataHelper.cloneSubContext(context); context.addIgnoredField( IgnoredSourceFieldMapper.NameValue.fromContext( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index fe1ad85d6a7c1..f47d86b746a38 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -24,13 +24,11 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.TreeSet; /** * Context used when parsing incoming documents. Holds everything that is needed to parse a document as well as @@ -106,7 +104,7 @@ public int get() { private final MappingParserContext mappingParserContext; private final SourceToParse sourceToParse; private final Set ignoredFields; - private final Set ignoredFieldValues; + private final List ignoredFieldValues; private final Map> dynamicMappers; private final DynamicMapperSize dynamicMappersSize; private final Map dynamicObjectMappers; @@ -128,7 +126,7 @@ private DocumentParserContext( MappingParserContext mappingParserContext, SourceToParse sourceToParse, Set ignoreFields, - Set ignoredFieldValues, + List ignoredFieldValues, Map> dynamicMappers, Map dynamicObjectMappers, Map> dynamicRuntimeFields, @@ -198,7 +196,7 @@ protected DocumentParserContext( mappingParserContext, source, new HashSet<>(), - new TreeSet<>(Comparator.comparing(IgnoredSourceFieldMapper.NameValue::name)), + new ArrayList<>(), new HashMap<>(), new HashMap<>(), new HashMap<>(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java index 6cf44ba6bc447..d8780f28b58a6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DotExpandingXContentParser.java @@ -50,6 +50,8 @@ private static final class WrappingParser extends FilterXContentParser { public Token nextToken() throws IOException { Token token; XContentParser delegate; + // cache object field (even when final this is a valid optimization, see https://openjdk.org/jeps/8132243) + var parsers = this.parsers; while ((token = (delegate = parsers.peek()).nextToken()) == null) { parsers.pop(); if (parsers.isEmpty()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 296e7df98b0cf..b31a61d50ecdb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.SimpleVectorTileFormatter; import org.elasticsearch.common.unit.DistanceUnit; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.IndexMode; @@ -45,6 +46,7 @@ import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.runtime.GeoPointScriptFieldDistanceFeatureQuery; +import org.elasticsearch.xcontent.CopyingXContentParser; import org.elasticsearch.xcontent.FilterXContentParserWrapper; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -193,13 +195,15 @@ private FieldValues scriptValues() { @Override public FieldMapper build(MapperBuilderContext context) { + boolean ignoreMalformedEnabled = ignoreMalformed.get().value(); Parser geoParser = new GeoPointParser( name(), (parser) -> GeoUtils.parseGeoPoint(parser, ignoreZValue.get().value()), nullValue.get(), ignoreZValue.get().value(), - ignoreMalformed.get().value(), - metric.get() != TimeSeriesParams.MetricType.POSITION + ignoreMalformedEnabled, + metric.get() != TimeSeriesParams.MetricType.POSITION, + context.isSourceSynthetic() && ignoreMalformedEnabled ); GeoPointFieldType ft = new GeoPointFieldType( context.buildFullName(name()), @@ -524,6 +528,7 @@ public TimeSeriesParams.MetricType getMetricType() { /** GeoPoint parser implementation */ private static class GeoPointParser extends PointParser { + private final boolean storeMalformedDataForSyntheticSource; GeoPointParser( String field, @@ -531,9 +536,11 @@ private static class GeoPointParser extends PointParser { GeoPoint nullValue, boolean ignoreZValue, boolean ignoreMalformed, - boolean allowMultipleValues + boolean allowMultipleValues, + boolean storeMalformedDataForSyntheticSource ) { super(field, objectParser, nullValue, ignoreZValue, ignoreMalformed, allowMultipleValues); + this.storeMalformedDataForSyntheticSource = storeMalformedDataForSyntheticSource; } protected GeoPoint validate(GeoPoint in) { @@ -568,6 +575,45 @@ public GeoPoint normalizeFromSource(GeoPoint point) { // normalize during parsing return point; } + + @Override + protected void parseAndConsumeFromObject( + XContentParser parser, + CheckedConsumer consumer, + MalformedValueHandler malformedHandler + ) throws IOException { + XContentParser parserWithCustomization = parser; + XContentBuilder malformedDataForSyntheticSource = null; + + if (storeMalformedDataForSyntheticSource) { + if (parser.currentToken() == XContentParser.Token.START_OBJECT + || parser.currentToken() == XContentParser.Token.START_ARRAY) { + // We have a complex structure so we'll memorize it while parsing. + var copyingParser = new CopyingXContentParser(parser); + malformedDataForSyntheticSource = copyingParser.getBuilder(); + parserWithCustomization = copyingParser; + } else { + // We have a single value (e.g. a string) that is potentially malformed, let's simply remember it. + malformedDataForSyntheticSource = XContentBuilder.builder(parser.contentType().xContent()).copyCurrentStructure(parser); + } + } + + try { + GeoPoint point = objectParser.apply(parserWithCustomization); + consumer.accept(validate(point)); + } catch (Exception e) { + malformedHandler.notify(e, malformedDataForSyntheticSource); + } + } + } + + @Override + protected void onMalformedValue(DocumentParserContext context, XContentBuilder malformedDataForSyntheticSource, Exception cause) + throws IOException { + super.onMalformedValue(context, malformedDataForSyntheticSource, cause); + if (malformedDataForSyntheticSource != null) { + context.doc().add(IgnoreMalformedStoredValues.storedField(name(), malformedDataForSyntheticSource)); + } } @Override @@ -585,11 +631,6 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values" ); } - if (ignoreMalformed()) { - throw new IllegalArgumentException( - "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed points" - ); - } if (copyTo.copyToFields().isEmpty() != true) { throw new IllegalArgumentException( "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeParser.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeParser.java index d20a700faff81..42f735a58cf51 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeParser.java @@ -18,7 +18,6 @@ import java.io.IOException; import java.text.ParseException; -import java.util.function.Consumer; public class GeoShapeParser extends AbstractGeometryFieldMapper.Parser { private final GeometryParser geometryParser; @@ -30,18 +29,21 @@ public GeoShapeParser(GeometryParser geometryParser, Orientation orientation) { } @Override - public void parse(XContentParser parser, CheckedConsumer consumer, Consumer onMalformed) - throws IOException { + public void parse( + XContentParser parser, + CheckedConsumer consumer, + AbstractGeometryFieldMapper.MalformedValueHandler malformedHandler + ) throws IOException { try { if (parser.currentToken() == XContentParser.Token.START_ARRAY) { while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - parse(parser, consumer, onMalformed); + parse(parser, consumer, malformedHandler); } } else { consumer.accept(geometryParser.parse(parser)); } } catch (ParseException | ElasticsearchParseException | IllegalArgumentException e) { - onMalformed.accept(e); + malformedHandler.notify(e); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java index 52f4048e9b230..3d2c51fb5b8af 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoreMalformedStoredValues.java @@ -25,10 +25,30 @@ * {@code _source}. */ public abstract class IgnoreMalformedStoredValues { + /** + * Creates a stored field that stores malformed data to be used in synthetic source. + * Name of the stored field is original name of the field with added conventional suffix. + * @param name original name of the field + * @param parser parser to grab field content from + * @return + * @throws IOException + */ public static StoredField storedField(String name, XContentParser parser) throws IOException { return XContentDataHelper.storedField(name(name), parser); } + /** + * Creates a stored field that stores malformed data to be used in synthetic source. + * Name of the stored field is original name of the field with added conventional suffix. + * @param name original name of the field + * @param builder malformed data + * @return + * @throws IOException + */ + public static StoredField storedField(String name, XContentBuilder builder) throws IOException { + return XContentDataHelper.storedField(name(name), builder); + } + /** * Build a {@link IgnoreMalformedStoredValues} that never contains any values. */ @@ -108,7 +128,7 @@ public void write(XContentBuilder b) throws IOException { } } - private static String name(String fieldName) { + public static String name(String fieldName) { return fieldName + "._ignore_malformed"; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java index 6e243e3575d37..f64511f8396ec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java @@ -17,7 +17,10 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; +import java.util.List; /** @@ -53,7 +56,7 @@ public class IgnoredSourceFieldMapper extends MetadataFieldMapper { * the full name of the parent field * - the value, encoded as a byte array */ - public record NameValue(String name, int parentOffset, BytesRef value) { + public record NameValue(String name, int parentOffset, BytesRef value, LuceneDocument doc) { /** * Factory method, for use with fields under the parent object. It doesn't apply to objects at root level. * @param context the parser context, containing a non-null parent @@ -62,7 +65,7 @@ public record NameValue(String name, int parentOffset, BytesRef value) { */ public static NameValue fromContext(DocumentParserContext context, String name, BytesRef value) { int parentOffset = context.parent() instanceof RootObjectMapper ? 0 : context.parent().fullPath().length() + 1; - return new NameValue(name, parentOffset, value); + return new NameValue(name, parentOffset, value, context.doc()); } String getParentFieldName() { @@ -112,8 +115,11 @@ protected String contentType() { public void postParse(DocumentParserContext context) { // Ignored values are only expected in synthetic mode. assert context.getIgnoredFieldValues().isEmpty() || context.mappingLookup().isSourceSynthetic(); - for (NameValue nameValue : context.getIgnoredFieldValues()) { - context.doc().add(new StoredField(NAME, encode(nameValue))); + List ignoredFieldValues = new ArrayList<>(context.getIgnoredFieldValues()); + // ensure consistent ordering when retrieving synthetic source + Collections.sort(ignoredFieldValues, Comparator.comparing(NameValue::name)); + for (NameValue nameValue : ignoredFieldValues) { + nameValue.doc().add(new StoredField(NAME, encode(nameValue))); } } @@ -136,7 +142,7 @@ static NameValue decode(Object field) { int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET; String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8); BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4); - return new NameValue(name, parentOffset, value); + return new NameValue(name, parentOffset, value, null); } // This mapper doesn't contribute to source directly as it has no access to the object structure. Instead, its contents diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 12098ef1170e7..ab5e731c1430a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -10,6 +10,7 @@ import org.elasticsearch.features.FeatureSpecification; import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import java.util.Set; @@ -23,7 +24,8 @@ public Set getFeatures() { IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE, PassThroughObjectMapper.PASS_THROUGH_PRIORITY, RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX, - SourceFieldMapper.SYNTHETIC_SOURCE_FALLBACK + SourceFieldMapper.SYNTHETIC_SOURCE_FALLBACK, + DenseVectorFieldMapper.INT4_QUANTIZATION ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 3ac4c0b0e18e1..277b9c4b66b33 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -560,18 +560,25 @@ public DocumentMapper merge(String type, CompressedXContent mappingSource, Merge return doMerge(type, reason, mappingSourceAsMap); } - private synchronized DocumentMapper doMerge(String type, MergeReason reason, Map mappingSourceAsMap) { + private DocumentMapper doMerge(String type, MergeReason reason, Map mappingSourceAsMap) { Mapping incomingMapping = parseMapping(type, reason, mappingSourceAsMap); - Mapping mapping = mergeMappings(this.mapper, incomingMapping, reason, this.indexSettings); // TODO: In many cases the source here is equal to mappingSource so we need not serialize again. // We should identify these cases reliably and save expensive serialization here - DocumentMapper newMapper = newDocumentMapper(mapping, reason, mapping.toCompressedXContent()); if (reason == MergeReason.MAPPING_AUTO_UPDATE_PREFLIGHT) { - return newMapper; + // only doing a merge without updating the actual #mapper field, no need to synchronize + Mapping mapping = mergeMappings(this.mapper, incomingMapping, MergeReason.MAPPING_AUTO_UPDATE_PREFLIGHT, this.indexSettings); + return newDocumentMapper(mapping, MergeReason.MAPPING_AUTO_UPDATE_PREFLIGHT, mapping.toCompressedXContent()); + } else { + // synchronized concurrent mapper updates are guaranteed to set merged mappers derived from the mapper value previously read + // TODO: can we even have concurrent updates here? + synchronized (this) { + Mapping mapping = mergeMappings(this.mapper, incomingMapping, reason, this.indexSettings); + DocumentMapper newMapper = newDocumentMapper(mapping, reason, mapping.toCompressedXContent()); + this.mapper = newMapper; + assert assertSerialization(newMapper, reason); + return newMapper; + } } - this.mapper = newMapper; - assert assertSerialization(newMapper, reason); - return newMapper; } private DocumentMapper newDocumentMapper(Mapping mapping, MergeReason reason, CompressedXContent mappingSource) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java index 65748847406ea..4bc633296a832 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java @@ -8,19 +8,31 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.join.BitSetProducer; +import org.apache.lucene.util.BitSet; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static org.elasticsearch.index.mapper.SourceFieldMetrics.NOOP; /** * A Mapper for nested objects @@ -34,12 +46,12 @@ public static class Builder extends ObjectMapper.Builder { private Explicit includeInRoot = Explicit.IMPLICIT_FALSE; private Explicit includeInParent = Explicit.IMPLICIT_FALSE; private final IndexVersion indexCreatedVersion; - private final Function bitsetProducer; + private final Function bitSetProducer; public Builder(String name, IndexVersion indexCreatedVersion, Function bitSetProducer) { super(name, Explicit.IMPLICIT_TRUE); this.indexCreatedVersion = indexCreatedVersion; - this.bitsetProducer = bitSetProducer; + this.bitSetProducer = bitSetProducer; } Builder includeInRoot(boolean includeInRoot) { @@ -91,12 +103,13 @@ public NestedObjectMapper build(MapperBuilderContext context) { buildMappers(nestedContext), enabled, dynamic, + storeArraySource, includeInParent, includeInRoot, parentTypeFilter, nestedTypePath, nestedTypeFilter, - bitsetProducer + bitSetProducer ); } } @@ -179,6 +192,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Map mappers, Explicit enabled, ObjectMapper.Dynamic dynamic, + Explicit storeArraySource, Explicit includeInParent, Explicit includeInRoot, Query parentTypeFilter, @@ -186,7 +200,7 @@ public MapperBuilderContext createChildContext(String name, Dynamic dynamic) { Query nestedTypeFilter, Function bitsetProducer ) { - super(name, fullPath, enabled, Explicit.IMPLICIT_TRUE, Explicit.IMPLICIT_FALSE, dynamic, mappers); + super(name, fullPath, enabled, Explicit.IMPLICIT_TRUE, storeArraySource, dynamic, mappers); this.parentTypeFilter = parentTypeFilter; this.nestedTypePath = nestedTypePath; this.nestedTypeFilter = nestedTypeFilter; @@ -246,6 +260,7 @@ NestedObjectMapper withoutMappers() { Map.of(), enabled, dynamic, + storeArraySource, includeInParent, includeInRoot, parentTypeFilter, @@ -271,6 +286,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (isEnabled() != Defaults.ENABLED) { builder.field("enabled", enabled.value()); } + if (storeArraySource != Defaults.STORE_ARRAY_SOURCE) { + builder.field(STORE_ARRAY_SOURCE_PARAM, storeArraySource.value()); + } serializeMappers(builder, params); return builder.endObject(); } @@ -317,6 +335,7 @@ public ObjectMapper merge(Mapper mergeWith, MapperMergeContext parentMergeContex mergeResult.mappers(), mergeResult.enabled(), mergeResult.dynamic(), + mergeResult.trackArraySource(), incInParent, incInRoot, parentTypeFilter, @@ -346,7 +365,111 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - // IgnoredSourceFieldMapper integration takes care of writing the source for nested objects. - return SourceLoader.SyntheticFieldLoader.NOTHING; + if (storeArraySource()) { + // IgnoredSourceFieldMapper integration takes care of writing the source for nested objects that enabled store_array_source. + return SourceLoader.SyntheticFieldLoader.NOTHING; + } + + SourceLoader sourceLoader = new SourceLoader.Synthetic(() -> super.syntheticFieldLoader(mappers.values().stream(), true), NOOP); + var storedFieldLoader = org.elasticsearch.index.fieldvisitor.StoredFieldLoader.create(false, sourceLoader.requiredStoredFields()); + return new NestedSyntheticFieldLoader( + storedFieldLoader, + sourceLoader, + () -> bitsetProducer.apply(parentTypeFilter), + nestedTypeFilter + ); + } + + private class NestedSyntheticFieldLoader implements SourceLoader.SyntheticFieldLoader { + private final org.elasticsearch.index.fieldvisitor.StoredFieldLoader storedFieldLoader; + private final SourceLoader sourceLoader; + private final Supplier parentBitSetProducer; + private final Query childFilter; + + private LeafStoredFieldLoader leafStoredFieldLoader; + private SourceLoader.Leaf leafSourceLoader; + private final List children = new ArrayList<>(); + + private NestedSyntheticFieldLoader( + org.elasticsearch.index.fieldvisitor.StoredFieldLoader storedFieldLoader, + SourceLoader sourceLoader, + Supplier parentBitSetProducer, + Query childFilter + ) { + this.storedFieldLoader = storedFieldLoader; + this.sourceLoader = sourceLoader; + this.parentBitSetProducer = parentBitSetProducer; + this.childFilter = childFilter; + } + + @Override + public Stream> storedFieldLoaders() { + return Stream.of(); + } + + @Override + public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { + this.children.clear(); + this.leafStoredFieldLoader = storedFieldLoader.getLoader(leafReader.getContext(), null); + this.leafSourceLoader = sourceLoader.leaf(leafReader, null); + + IndexSearcher searcher = new IndexSearcher(leafReader); + searcher.setQueryCache(null); + var childScorer = searcher.createWeight(childFilter, ScoreMode.COMPLETE_NO_SCORES, 1f).scorer(leafReader.getContext()); + if (childScorer != null) { + var parentDocs = parentBitSetProducer.get().getBitSet(leafReader.getContext()); + return parentDoc -> { + collectChildren(parentDoc, parentDocs, childScorer.iterator()); + return children.size() > 0; + }; + } else { + return parentDoc -> false; + } + } + + private List collectChildren(int parentDoc, BitSet parentDocs, DocIdSetIterator childIt) throws IOException { + assert parentDocs.get(parentDoc) : "wrong context, doc " + parentDoc + " is not a parent of " + nestedTypePath; + final int prevParentDoc = parentDocs.prevSetBit(parentDoc - 1); + int childDocId = childIt.docID(); + if (childDocId <= prevParentDoc) { + childDocId = childIt.advance(prevParentDoc + 1); + } + + children.clear(); + for (; childDocId < parentDoc; childDocId = childIt.nextDoc()) { + children.add(childDocId); + } + return children; + } + + @Override + public boolean hasValue() { + return children.size() > 0; + } + + @Override + public void write(XContentBuilder b) throws IOException { + assert (children != null && children.size() > 0); + if (children.size() == 1) { + b.startObject(simpleName()); + leafStoredFieldLoader.advanceTo(children.get(0)); + leafSourceLoader.write(leafStoredFieldLoader, children.get(0), b); + b.endObject(); + } else { + b.startArray(simpleName()); + for (int childId : children) { + b.startObject(); + leafStoredFieldLoader.advanceTo(childId); + leafSourceLoader.write(leafStoredFieldLoader, childId, b); + b.endObject(); + } + b.endArray(); + } + } + + @Override + public String fieldName() { + return name(); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index b21b77bc86dd8..356c103756bac 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -756,12 +756,16 @@ protected void doXContent(XContentBuilder builder, Params params) throws IOExcep } - public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream mappers) { + protected SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream mappers, boolean isFragment) { var fields = mappers.sorted(Comparator.comparing(Mapper::name)) .map(Mapper::syntheticFieldLoader) .filter(l -> l != SourceLoader.SyntheticFieldLoader.NOTHING) .toList(); - return new SyntheticSourceFieldLoader(fields); + return new SyntheticSourceFieldLoader(fields, isFragment); + } + + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader(Stream mappers) { + return syntheticFieldLoader(mappers, false); } @Override @@ -771,11 +775,13 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { private class SyntheticSourceFieldLoader implements SourceLoader.SyntheticFieldLoader { private final List fields; + private final boolean isFragment; private boolean hasValue; private List ignoredValues; - private SyntheticSourceFieldLoader(List fields) { + private SyntheticSourceFieldLoader(List fields, boolean isFragment) { this.fields = fields; + this.isFragment = isFragment; } @Override @@ -830,18 +836,21 @@ public void write(XContentBuilder b) throws IOException { if (hasValue == false) { return; } - if (isRoot()) { - if (isEnabled() == false) { - // If the root object mapper is disabled, it is expected to contain - // the source encapsulated within a single ignored source value. - assert ignoredValues.size() == 1 : ignoredValues.size(); - XContentDataHelper.decodeAndWrite(b, ignoredValues.get(0).value()); - ignoredValues = null; - return; + if (isRoot() && isEnabled() == false) { + // If the root object mapper is disabled, it is expected to contain + // the source encapsulated within a single ignored source value. + assert ignoredValues.size() == 1 : ignoredValues.size(); + XContentDataHelper.decodeAndWrite(b, ignoredValues.get(0).value()); + ignoredValues = null; + return; + } + + if (isFragment == false) { + if (isRoot()) { + b.startObject(); + } else { + b.startObject(simpleName()); } - b.startObject(); - } else { - b.startObject(simpleName()); } if (ignoredValues != null && ignoredValues.isEmpty() == false) { @@ -868,7 +877,9 @@ public void write(XContentBuilder b) throws IOException { } } hasValue = false; - b.endObject(); + if (isFragment == false) { + b.endObject(); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java index 6b5b2537e5e1f..254a0bc9c906b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/XContentDataHelper.java @@ -45,6 +45,13 @@ static StoredField storedField(String name, XContentParser parser) throws IOExce return (StoredField) processToken(parser, typeUtils -> typeUtils.buildStoredField(name, parser)); } + /** + * Build a {@link StoredField} for the value provided in a {@link XContentBuilder}. + */ + static StoredField storedField(String name, XContentBuilder builder) throws IOException { + return new StoredField(name, TypeUtils.encode(builder)); + } + /** * Build a {@link BytesRef} wrapping a byte array containing an encoded form * the value on which the parser is currently positioned. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 8a3c126f93a09..88772169e260c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -44,6 +44,7 @@ import org.apache.lucene.util.VectorUtil; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.codec.vectors.ES813FlatVectorFormat; @@ -106,6 +107,8 @@ static boolean isNotUnitVector(float magnitude) { return Math.abs(magnitude - 1.0f) > EPS; } + public static final NodeFeature INT4_QUANTIZATION = new NodeFeature("mapper.vectors.int4_quantization"); + public static final IndexVersion MAGNITUDE_STORED_INDEX_VERSION = IndexVersions.V_7_5_0; public static final IndexVersion INDEXED_BY_DEFAULT_INDEX_VERSION = IndexVersions.FIRST_DETACHED_INDEX_VERSION; public static final IndexVersion NORMALIZE_COSINE = IndexVersions.NORMALIZED_VECTOR_COSINE; @@ -198,6 +201,9 @@ public Builder(String name, IndexVersion indexVersionCreated) { }, Objects::toString ).setSerializerCheck((id, ic, v) -> v != null).addValidator(v -> { + if (v != null && dims.isConfigured() && dims.get() != null) { + v.validateDimension(dims.get()); + } if (v != null && v.supportsElementType(elementType.getValue()) == false) { throw new IllegalArgumentException( "[element_type] cannot be [" + elementType.getValue().toString() + "] when using index type [" + v.type + "]" @@ -255,6 +261,7 @@ public DenseVectorFieldMapper build(MapperBuilderContext context) { dims.getValue(), indexed.getValue(), similarity.getValue(), + indexOptions.getValue(), meta.getValue() ), indexOptions.getValue(), @@ -857,7 +864,7 @@ public final String toString() { public abstract VectorSimilarityFunction vectorSimilarityFunction(IndexVersion indexVersion, ElementType elementType); } - private abstract static class IndexOptions implements ToXContent { + abstract static class IndexOptions implements ToXContent { final String type; IndexOptions(String type) { @@ -871,6 +878,10 @@ boolean supportsElementType(ElementType elementType) { } abstract boolean updatableTo(IndexOptions update); + + void validateDimension(int dim) { + // no-op + } } private enum VectorIndexType { @@ -913,6 +924,27 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti return new Int8HnswIndexOptions(m, efConstruction, confidenceInterval); } }, + INT4_HNSW("int4_hnsw") { + public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { + Object mNode = indexOptionsMap.remove("m"); + Object efConstructionNode = indexOptionsMap.remove("ef_construction"); + Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); + if (mNode == null) { + mNode = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + } + if (efConstructionNode == null) { + efConstructionNode = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + } + int m = XContentMapValues.nodeIntegerValue(mNode); + int efConstruction = XContentMapValues.nodeIntegerValue(efConstructionNode); + Float confidenceInterval = null; + if (confidenceIntervalNode != null) { + confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode); + } + MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new Int4HnswIndexOptions(m, efConstruction, confidenceInterval); + } + }, FLAT("flat") { @Override public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { @@ -929,7 +961,19 @@ public IndexOptions parseIndexOptions(String fieldName, Map indexOpti confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode); } MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); - return new Int8FlatIndexOption(confidenceInterval); + return new Int8FlatIndexOptions(confidenceInterval); + } + }, + INT4_FLAT("int4_flat") { + @Override + public IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap) { + Object confidenceIntervalNode = indexOptionsMap.remove("confidence_interval"); + Float confidenceInterval = null; + if (confidenceIntervalNode != null) { + confidenceInterval = (float) XContentMapValues.nodeDoubleValue(confidenceIntervalNode); + } + MappingParser.checkNoRemainingFields(fieldName, indexOptionsMap); + return new Int4FlatIndexOptions(confidenceInterval); } }; @@ -946,10 +990,10 @@ static Optional fromString(String type) { abstract IndexOptions parseIndexOptions(String fieldName, Map indexOptionsMap); } - private static class Int8FlatIndexOption extends IndexOptions { + static class Int8FlatIndexOptions extends IndexOptions { private final Float confidenceInterval; - Int8FlatIndexOption(Float confidenceInterval) { + Int8FlatIndexOptions(Float confidenceInterval) { super("int8_flat"); this.confidenceInterval = confidenceInterval; } @@ -967,14 +1011,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override KnnVectorsFormat getVectorsFormat() { - return new ES813Int8FlatVectorFormat(confidenceInterval); + return new ES813Int8FlatVectorFormat(confidenceInterval, 7, false); } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - Int8FlatIndexOption that = (Int8FlatIndexOption) o; + Int8FlatIndexOptions that = (Int8FlatIndexOptions) o; return Objects.equals(confidenceInterval, that.confidenceInterval); } @@ -996,7 +1040,7 @@ boolean updatableTo(IndexOptions update) { } } - private static class FlatIndexOptions extends IndexOptions { + static class FlatIndexOptions extends IndexOptions { FlatIndexOptions() { super("flat"); @@ -1032,12 +1076,147 @@ public int hashCode() { } } - private static class Int8HnswIndexOptions extends IndexOptions { + static class Int4HnswIndexOptions extends IndexOptions { + private final int m; + private final int efConstruction; + private final float confidenceInterval; + + Int4HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) { + super("int4_hnsw"); + this.m = m; + this.efConstruction = efConstruction; + // The default confidence interval for int4 is dynamic quantiles, this provides the best relevancy and is + // effectively required for int4 to behave well across a wide range of data. + this.confidenceInterval = confidenceInterval == null ? 0f : confidenceInterval; + } + + @Override + public KnnVectorsFormat getVectorsFormat() { + return new ES814HnswScalarQuantizedVectorsFormat(m, efConstruction, confidenceInterval, 4, true); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + builder.field("m", m); + builder.field("ef_construction", efConstruction); + builder.field("confidence_interval", confidenceInterval); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Int4HnswIndexOptions that = (Int4HnswIndexOptions) o; + return m == that.m && efConstruction == that.efConstruction && Objects.equals(confidenceInterval, that.confidenceInterval); + } + + @Override + public int hashCode() { + return Objects.hash(m, efConstruction, confidenceInterval); + } + + @Override + public String toString() { + return "{type=" + + type + + ", m=" + + m + + ", ef_construction=" + + efConstruction + + ", confidence_interval=" + + confidenceInterval + + "}"; + } + + @Override + boolean supportsElementType(ElementType elementType) { + return elementType != ElementType.BYTE; + } + + @Override + boolean updatableTo(IndexOptions update) { + return Objects.equals(this, update); + } + + @Override + void validateDimension(int dim) { + if (dim % 2 != 0) { + throw new IllegalArgumentException("int4_hnsw only supports even dimensions; provided=" + dim); + } + } + } + + static class Int4FlatIndexOptions extends IndexOptions { + private final float confidenceInterval; + + Int4FlatIndexOptions(Float confidenceInterval) { + super("int4_flat"); + // The default confidence interval for int4 is dynamic quantiles, this provides the best relevancy and is + // effectively required for int4 to behave well across a wide range of data. + this.confidenceInterval = confidenceInterval == null ? 0f : confidenceInterval; + } + + @Override + public KnnVectorsFormat getVectorsFormat() { + return new ES813Int8FlatVectorFormat(confidenceInterval, 4, true); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("type", type); + builder.field("confidence_interval", confidenceInterval); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Int4FlatIndexOptions that = (Int4FlatIndexOptions) o; + return Objects.equals(confidenceInterval, that.confidenceInterval); + } + + @Override + public int hashCode() { + return Objects.hash(confidenceInterval); + } + + @Override + public String toString() { + return "{type=" + type + ", confidence_interval=" + confidenceInterval + "}"; + } + + @Override + boolean supportsElementType(ElementType elementType) { + return elementType != ElementType.BYTE; + } + + @Override + boolean updatableTo(IndexOptions update) { + // TODO: add support for updating from flat, hnsw, and int8_hnsw and updating params + return Objects.equals(this, update); + } + + @Override + void validateDimension(int dim) { + if (dim % 2 != 0) { + throw new IllegalArgumentException("int4_flat only supports even dimensions; provided=" + dim); + } + } + } + + static class Int8HnswIndexOptions extends IndexOptions { private final int m; private final int efConstruction; private final Float confidenceInterval; - private Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) { + Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval) { super("int8_hnsw"); this.m = m; this.efConstruction = efConstruction; @@ -1046,9 +1225,7 @@ private Int8HnswIndexOptions(int m, int efConstruction, Float confidenceInterval @Override public KnnVectorsFormat getVectorsFormat() { - // int bits = 7; - // boolean compress = false; // TODO we only support 7 and false, for now - return new ES814HnswScalarQuantizedVectorsFormat(m, efConstruction, 1, confidenceInterval, null); + return new ES814HnswScalarQuantizedVectorsFormat(m, efConstruction, confidenceInterval, 7, false); } @Override @@ -1111,11 +1288,11 @@ boolean updatableTo(IndexOptions update) { } } - private static class HnswIndexOptions extends IndexOptions { + static class HnswIndexOptions extends IndexOptions { private final int m; private final int efConstruction; - private HnswIndexOptions(int m, int efConstruction) { + HnswIndexOptions(int m, int efConstruction) { super("hnsw"); this.m = m; this.efConstruction = efConstruction; @@ -1177,6 +1354,7 @@ public static final class DenseVectorFieldType extends SimpleMappedFieldType { private final boolean indexed; private final VectorSimilarity similarity; private final IndexVersion indexVersionCreated; + private final IndexOptions indexOptions; public DenseVectorFieldType( String name, @@ -1185,6 +1363,7 @@ public DenseVectorFieldType( Integer dims, boolean indexed, VectorSimilarity similarity, + IndexOptions indexOptions, Map meta ) { super(name, indexed, false, indexed == false, TextSearchInfo.NONE, meta); @@ -1193,6 +1372,7 @@ public DenseVectorFieldType( this.indexed = indexed; this.similarity = similarity; this.indexVersionCreated = indexVersionCreated; + this.indexOptions = indexOptions; } @Override @@ -1456,6 +1636,10 @@ int getVectorDimensions() { ElementType getElementType() { return elementType; } + + IndexOptions getIndexOptions() { + return indexOptions; + } } private final IndexOptions indexOptions; @@ -1500,6 +1684,9 @@ public void parse(DocumentParserContext context) throws IOException { } if (fieldType().dims == null) { int dims = fieldType().elementType.parseDimensionCount(context); + if (fieldType().indexOptions != null) { + fieldType().indexOptions.validateDimension(dims); + } DenseVectorFieldType updatedDenseVectorFieldType = new DenseVectorFieldType( fieldType().name(), indexCreatedVersion, @@ -1507,6 +1694,7 @@ public void parse(DocumentParserContext context) throws IOException { dims, fieldType().indexed, fieldType().similarity, + fieldType().indexOptions, fieldType().meta() ); Mapper update = new DenseVectorFieldMapper( diff --git a/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java b/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java index 20a7ffe9c7433..69aed030166c4 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/DocsStats.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; @@ -81,7 +82,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.DOCS); builder.field(Fields.COUNT, count); builder.field(Fields.DELETED, deleted); - builder.field(Fields.TOTAL_SIZE_IN_BYTES, totalSizeInBytes); + builder.humanReadableField(Fields.TOTAL_SIZE_IN_BYTES, Fields.TOTAL_SIZE, ByteSizeValue.ofBytes(totalSizeInBytes)); builder.endObject(); return builder; } @@ -104,5 +105,6 @@ static final class Fields { static final String COUNT = "count"; static final String DELETED = "deleted"; static final String TOTAL_SIZE_IN_BYTES = "total_size_in_bytes"; + static final String TOTAL_SIZE = "total_size"; } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 39044720bea16..b3f19b1b7a81d 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -1428,6 +1428,12 @@ public DenseVectorStats denseVectorStats() { return getEngine().denseVectorStats(); } + public SparseVectorStats sparseVectorStats() { + readAllowed(); + MappingLookup mappingLookup = mapperService != null ? mapperService.mappingLookup() : null; + return getEngine().sparseVectorStats(mappingLookup); + } + public BulkStats bulkStats() { return bulkOperationListener.stats(); } @@ -3485,7 +3491,8 @@ private EngineConfig newEngineConfig(LongSupplier globalCheckpointSupplier) { isTimeBasedIndex ? TIMESERIES_LEAF_READERS_SORTER : null, relativeTimeInNanosSupplier, indexCommitListener, - routingEntry().isPromotableToPrimary() + routingEntry().isPromotableToPrimary(), + mapperService() ); } diff --git a/server/src/main/java/org/elasticsearch/index/shard/SparseVectorStats.java b/server/src/main/java/org/elasticsearch/index/shard/SparseVectorStats.java new file mode 100644 index 0000000000000..738d38eb65620 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/shard/SparseVectorStats.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.ToXContentFragment; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Statistics about indexed sparse vector + */ +public class SparseVectorStats implements Writeable, ToXContentFragment { + private long valueCount = 0; + + public SparseVectorStats() {} + + public SparseVectorStats(long count) { + this.valueCount = count; + } + + public SparseVectorStats(StreamInput in) throws IOException { + this.valueCount = in.readVLong(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(valueCount); + } + + public void add(SparseVectorStats other) { + if (other == null) { + return; + } + this.valueCount += other.valueCount; + } + + /** + * Returns the total number of dense vectors added in the index. + */ + public long getValueCount() { + return valueCount; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(Fields.NAME); + builder.field(Fields.VALUE_COUNT, valueCount); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SparseVectorStats that = (SparseVectorStats) o; + return valueCount == that.valueCount; + } + + @Override + public int hashCode() { + return Objects.hash(valueCount); + } + + static final class Fields { + static final String NAME = "sparse_vector"; + static final String VALUE_COUNT = "value_count"; + } +} diff --git a/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java b/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java index 9ce2bc201c20f..0e90e907efa4c 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java +++ b/server/src/main/java/org/elasticsearch/indices/IndexingMemoryController.java @@ -90,7 +90,7 @@ public class IndexingMemoryController implements IndexingOperationListener, Clos private final Iterable indexShards; - private final ByteSizeValue indexingBuffer; + private final long indexingBuffer; private final TimeValue inactiveTime; private final TimeValue interval; @@ -129,7 +129,7 @@ public class IndexingMemoryController implements IndexingOperationListener, Clos indexingBuffer = maxIndexingBuffer; } } - this.indexingBuffer = indexingBuffer; + this.indexingBuffer = indexingBuffer.getBytes(); this.inactiveTime = SHARD_INACTIVE_TIME_SETTING.get(settings); // we need to have this relatively small to free up heap quickly enough @@ -165,7 +165,7 @@ public void close() { * returns the current budget for the total amount of indexing buffers of * active shards on this node */ - ByteSizeValue indexingBufferSize() { + long indexingBufferSize() { return indexingBuffer; } @@ -295,13 +295,13 @@ final class ShardsIndicesStatusChecker implements Runnable { public void bytesWritten(int bytes) { long totalBytes = bytesWrittenSinceCheck.addAndGet(bytes); assert totalBytes >= 0; - while (totalBytes > indexingBuffer.getBytes() / 128) { + while (totalBytes > indexingBuffer / 128) { if (runLock.tryLock()) { try { // Must pull this again because it may have changed since we first checked: totalBytes = bytesWrittenSinceCheck.get(); - if (totalBytes > indexingBuffer.getBytes() / 128) { + if (totalBytes > indexingBuffer / 128) { bytesWrittenSinceCheck.addAndGet(-totalBytes); // NOTE: this is only an approximate check, because bytes written is to the translog, // vs indexing memory buffer which is typically smaller but can be larger in extreme @@ -393,9 +393,9 @@ private void runUnlocked() { // If we are using more than 50% of our budget across both indexing buffer and bytes we are still moving to disk, then we now // throttle the top shards to send back-pressure to ongoing indexing: - boolean doThrottle = (totalBytesWriting + totalBytesUsed) > 1.5 * indexingBuffer.getBytes(); + boolean doThrottle = (totalBytesWriting + totalBytesUsed) > 1.5 * indexingBuffer; - if (totalBytesUsed > indexingBuffer.getBytes()) { + if (totalBytesUsed > indexingBuffer) { // OK we are now over-budget; fill the priority queue and ask largest shard(s) to refresh: List queue = new ArrayList<>(); @@ -487,7 +487,7 @@ private void runUnlocked() { throttled.add(shardAndBytesUsed.shard); activateThrottling(shardAndBytesUsed.shard); } - if (totalBytesUsed <= indexingBuffer.getBytes()) { + if (totalBytesUsed <= indexingBuffer) { break; } } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index c0483ee2c8208..199bbc54fa3d6 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -56,7 +56,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.util.concurrent.AbstractRunnable; @@ -1626,7 +1625,7 @@ public void loadIntoContext(ShardSearchRequest request, SearchContext context) t } } - public ByteSizeValue getTotalIndexingBufferBytes() { + public long getTotalIndexingBufferBytes() { return indexingMemoryController.indexingBufferSize(); } diff --git a/server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java b/server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java index 2fe5e80d47b2b..6e898abb77e7f 100644 --- a/server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java +++ b/server/src/main/java/org/elasticsearch/indices/NodeIndicesStats.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.shard.DocsStats; import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.index.shard.ShardCountStats; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.index.warmer.WarmerStats; @@ -214,6 +215,11 @@ public DenseVectorStats getDenseVectorStats() { return stats.getDenseVectorStats(); } + @Nullable + public SparseVectorStats getSparseVectorStats() { + return stats.getSparseVectorStats(); + } + @Override public void writeTo(StreamOutput out) throws IOException { stats.writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java b/server/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java index 9cf5851454d6c..2ddfa9a3c1755 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/StartRecoveryRequest.java @@ -174,4 +174,26 @@ public void writeTo(StreamOutput out) throws IOException { out.writeBoolean(canDownloadSnapshotFiles); } } + + @Override + public String toString() { + return "StartRecoveryRequest{" + + "shardId=" + + shardId + + ", targetNode=" + + targetNode.descriptionWithoutAttributes() + + ", recoveryId=" + + recoveryId + + ", targetAllocationId='" + + targetAllocationId + + "', clusterStateVersion=" + + clusterStateVersion + + ", primaryRelocation=" + + primaryRelocation + + ", startingSeqNo=" + + startingSeqNo + + ", canDownloadSnapshotFiles=" + + canDownloadSnapshotFiles + + '}'; + } } diff --git a/server/src/main/java/org/elasticsearch/indices/recovery/StatelessPrimaryRelocationAction.java b/server/src/main/java/org/elasticsearch/indices/recovery/StatelessPrimaryRelocationAction.java index bdc7f5b2aafce..46908fbeec107 100644 --- a/server/src/main/java/org/elasticsearch/indices/recovery/StatelessPrimaryRelocationAction.java +++ b/server/src/main/java/org/elasticsearch/indices/recovery/StatelessPrimaryRelocationAction.java @@ -102,5 +102,21 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(recoveryId, shardId, targetNode, targetAllocationId, clusterStateVersion); } + + @Override + public String toString() { + return "Request{" + + "shardId=" + + shardId + + ", targetNode=" + + targetNode.descriptionWithoutAttributes() + + ", recoveryId=" + + recoveryId + + ", targetAllocationId='" + + targetAllocationId + + "', clusterStateVersion=" + + clusterStateVersion + + '}'; + } } } diff --git a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java index 9e6d066d38c7c..532aca07e5513 100644 --- a/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java +++ b/server/src/main/java/org/elasticsearch/indices/store/TransportNodesListShardStoreMetadata.java @@ -301,11 +301,6 @@ public ShardId shardId() { public String getCustomDataPath() { return customDataPath; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } } public static class NodesStoreFilesMetadata extends BaseNodesResponse { diff --git a/server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java b/server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java index ae454b6af1e6c..edffaff894eda 100644 --- a/server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java +++ b/server/src/main/java/org/elasticsearch/monitor/jvm/HotThreads.java @@ -15,6 +15,9 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.ReferenceDocs; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.ChunkedLoggingStream; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.unit.ByteSizeValue; @@ -22,6 +25,7 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.transport.Transports; +import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; import java.lang.management.ManagementFactory; @@ -550,4 +554,43 @@ public static void initializeRuntimeMonitoring() { } } } + + public record RequestOptions( + int threads, + HotThreads.ReportType reportType, + HotThreads.SortOrder sortOrder, + TimeValue interval, + int snapshots, + boolean ignoreIdleThreads + ) implements Writeable { + + public static RequestOptions readFrom(StreamInput in) throws IOException { + var threads = in.readInt(); + var ignoreIdleThreads = in.readBoolean(); + var reportType = HotThreads.ReportType.of(in.readString()); + var interval = in.readTimeValue(); + var snapshots = in.readInt(); + var sortOrder = HotThreads.SortOrder.of(in.readString()); + return new RequestOptions(threads, reportType, sortOrder, interval, snapshots, ignoreIdleThreads); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(threads); + out.writeBoolean(ignoreIdleThreads); + out.writeString(reportType.getTypeValue()); + out.writeTimeValue(interval); + out.writeInt(snapshots); + out.writeString(sortOrder.getOrderValue()); + } + + public static final RequestOptions DEFAULT = new RequestOptions( + 3, + ReportType.CPU, + SortOrder.TOTAL, + TimeValue.timeValueMillis(500), + 10, + true + ); + } } diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java index fd2aabce8e952..bcf8451e5fe54 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java +++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java @@ -221,7 +221,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Function; -import java.util.function.Supplier; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -671,7 +670,27 @@ private void construct( SystemIndices systemIndices = createSystemIndices(settings); - final SetOnce repositoriesServiceReference = new SetOnce<>(); + CircuitBreakerService circuitBreakerService = createCircuitBreakerService( + new CircuitBreakerMetrics(telemetryProvider), + settingsModule.getSettings(), + settingsModule.getClusterSettings() + ); + PageCacheRecycler pageCacheRecycler = serviceProvider.newPageCacheRecycler(pluginsService, settings); + BigArrays bigArrays = serviceProvider.newBigArrays(pluginsService, pageCacheRecycler, circuitBreakerService); + + final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); + RepositoriesModule repositoriesModule = new RepositoriesModule( + environment, + pluginsService.filterPlugins(RepositoryPlugin.class).toList(), + client, + threadPool, + clusterService, + bigArrays, + xContentRegistry, + recoverySettings, + telemetryProvider + ); + RepositoriesService repositoriesService = repositoriesModule.getRepositoryService(); final SetOnce rerouteServiceReference = new SetOnce<>(); final ClusterInfoService clusterInfoService = serviceProvider.newClusterInfoService( pluginsService, @@ -683,7 +702,7 @@ private void construct( final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( settings, clusterService, - repositoriesServiceReference::get, + repositoriesService, rerouteServiceReference::get ); final ClusterModule clusterModule = new ClusterModule( @@ -716,11 +735,6 @@ private void construct( IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class).toList()); modules.add(indicesModule); - CircuitBreakerService circuitBreakerService = createCircuitBreakerService( - new CircuitBreakerMetrics(telemetryProvider), - settingsModule.getSettings(), - settingsModule.getClusterSettings() - ); modules.add(new GatewayModule()); CompatibilityVersions compatibilityVersions = new CompatibilityVersions( @@ -729,8 +743,6 @@ private void construct( ); modules.add(loadPersistedClusterStateService(clusterService.getClusterSettings(), threadPool, compatibilityVersions)); - PageCacheRecycler pageCacheRecycler = serviceProvider.newPageCacheRecycler(pluginsService, settings); - BigArrays bigArrays = serviceProvider.newBigArrays(pluginsService, pageCacheRecycler, circuitBreakerService); final MetaStateService metaStateService = new MetaStateService(nodeEnvironment, xContentRegistry); FeatureService featureService = new FeatureService(pluginsService.loadServiceProviders(FeatureSpecification.class)); @@ -820,7 +832,7 @@ record PluginServiceInstances( NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry, IndexNameExpressionResolver indexNameExpressionResolver, - Supplier repositoriesServiceSupplier, + RepositoriesService repositoriesService, TelemetryProvider telemetryProvider, AllocationService allocationService, IndicesService indicesService, @@ -841,7 +853,7 @@ record PluginServiceInstances( nodeEnvironment, namedWriteableRegistry, clusterModule.getIndexNameExpressionResolver(), - repositoriesServiceReference::get, + repositoriesService, telemetryProvider, clusterModule.getAllocationService(), indicesService, @@ -948,25 +960,12 @@ record PluginServiceInstances( final HttpServerTransport httpServerTransport = serviceProvider.newHttpTransport(pluginsService, networkModule); final IndexingPressure indexingLimits = new IndexingPressure(settings); - final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings()); - RepositoriesModule repositoriesModule = new RepositoriesModule( - environment, - pluginsService.filterPlugins(RepositoryPlugin.class).toList(), - transportService, - clusterService, - bigArrays, - xContentRegistry, - recoverySettings, - telemetryProvider - ); - RepositoriesService repositoryService = repositoriesModule.getRepositoryService(); - repositoriesServiceReference.set(repositoryService); SnapshotsService snapshotsService = new SnapshotsService( settings, clusterService, rerouteService, clusterModule.getIndexNameExpressionResolver(), - repositoryService, + repositoriesService, transportService, actionModule.getActionFilters(), systemIndices @@ -974,12 +973,12 @@ record PluginServiceInstances( SnapshotShardsService snapshotShardsService = new SnapshotShardsService( settings, clusterService, - repositoryService, + repositoriesService, transportService, indicesService ); - actionModule.getReservedClusterStateService().installStateHandler(new ReservedRepositoryAction(repositoryService)); + actionModule.getReservedClusterStateService().installStateHandler(new ReservedRepositoryAction(repositoriesService)); actionModule.getReservedClusterStateService().installStateHandler(new ReservedPipelineAction()); FileSettingsService fileSettingsService = new FileSettingsService( @@ -990,7 +989,7 @@ record PluginServiceInstances( RestoreService restoreService = new RestoreService( clusterService, - repositoryService, + repositoriesService, clusterModule.getAllocationService(), metadataCreateIndexService, indexMetadataVerifier, @@ -1031,7 +1030,7 @@ record PluginServiceInstances( searchTransportService, indexingLimits, searchModule.getValuesSourceRegistry().getUsageService(), - repositoryService + repositoriesService ); final TimeValue metricsInterval = settings.getAsTime("telemetry.agent.metrics_interval", TimeValue.timeValueSeconds(10)); @@ -1075,14 +1074,14 @@ record PluginServiceInstances( featureService, threadPool, telemetryProvider, - repositoryService + repositoriesService ) ); - RecoveryPlannerService recoveryPlannerService = getRecoveryPlannerService(threadPool, clusterService, repositoryService); + RecoveryPlannerService recoveryPlannerService = getRecoveryPlannerService(threadPool, clusterService, repositoriesService); modules.add(b -> { serviceProvider.processRecoverySettings(pluginsService, settingsModule.getClusterSettings(), recoverySettings); - SnapshotFilesProvider snapshotFilesProvider = new SnapshotFilesProvider(repositoryService); + SnapshotFilesProvider snapshotFilesProvider = new SnapshotFilesProvider(repositoriesService); var peerRecovery = new PeerRecoverySourceService( transportService, indicesService, @@ -1140,7 +1139,7 @@ record PluginServiceInstances( b.bind(SnapshotsInfoService.class).toInstance(snapshotsInfoService); b.bind(FeatureService.class).toInstance(featureService); b.bind(HttpServerTransport.class).toInstance(httpServerTransport); - b.bind(RepositoriesService.class).toInstance(repositoryService); + b.bind(RepositoriesService.class).toInstance(repositoriesService); b.bind(SnapshotsService.class).toInstance(snapshotsService); b.bind(SnapshotShardsService.class).toInstance(snapshotShardsService); b.bind(RestoreService.class).toInstance(restoreService); @@ -1409,12 +1408,12 @@ private static ReloadablePlugin wrapPlugins(List reloadablePlu private RecoveryPlannerService getRecoveryPlannerService( ThreadPool threadPool, ClusterService clusterService, - RepositoriesService repositoryService + RepositoriesService repositoriesService ) { var recoveryPlannerServices = pluginsService.filterPlugins(RecoveryPlannerPlugin.class) .map( plugin -> plugin.createRecoveryPlannerService( - new ShardSnapshotsService(client, repositoryService, threadPool, clusterService) + new ShardSnapshotsService(client, repositoriesService, threadPool, clusterService) ) ) .flatMap(Optional::stream); diff --git a/server/src/main/java/org/elasticsearch/node/NodeService.java b/server/src/main/java/org/elasticsearch/node/NodeService.java index 87384b50d7ffd..059b05091a6ae 100644 --- a/server/src/main/java/org/elasticsearch/node/NodeService.java +++ b/server/src/main/java/org/elasticsearch/node/NodeService.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; @@ -138,7 +139,7 @@ public NodeInfo info( plugin ? (pluginService == null ? null : pluginService.info()) : null, ingest ? (ingestService == null ? null : ingestService.info()) : null, aggs ? (aggregationUsageService == null ? null : aggregationUsageService.info()) : null, - indices ? indicesService.getTotalIndexingBufferBytes() : null + indices ? ByteSizeValue.ofBytes(indicesService.getTotalIndexingBufferBytes()) : null ); } diff --git a/server/src/main/java/org/elasticsearch/plugins/Plugin.java b/server/src/main/java/org/elasticsearch/plugins/Plugin.java index 3003d11bf7c69..316dd37c2b029 100644 --- a/server/src/main/java/org/elasticsearch/plugins/Plugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/Plugin.java @@ -44,7 +44,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.function.Supplier; import java.util.function.UnaryOperator; /** @@ -127,11 +126,9 @@ public interface PluginServices { IndexNameExpressionResolver indexNameExpressionResolver(); /** - * A supplier for the service that manages snapshot repositories. - * This will return null when {@link #createComponents(PluginServices)} is called, - * but will return the repositories service once the node is initialized. + * A service that manages snapshot repositories. */ - Supplier repositoriesServiceSupplier(); + RepositoriesService repositoriesService(); /** * An interface for distributed tracing diff --git a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java index 0e404ca03707f..6fe1e48b25272 100644 --- a/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java +++ b/server/src/main/java/org/elasticsearch/plugins/internal/DocumentParsingProvider.java @@ -8,7 +8,7 @@ package org.elasticsearch.plugins.internal; -import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.mapper.MapperService; /** * An interface to provide instances of document parsing observer and reporter @@ -36,7 +36,7 @@ default DocumentSizeObserver newFixedSizeDocumentObserver(long normalisedBytesPa */ default DocumentSizeReporter newDocumentSizeReporter( String indexName, - IndexMode indexMode, + MapperService mapperService, DocumentSizeAccumulator documentSizeAccumulator ) { return DocumentSizeReporter.EMPTY_INSTANCE; diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java index 50aa7881cd2b6..85f06580cee79 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesMetrics.java @@ -16,6 +16,7 @@ public record RepositoriesMetrics( MeterRegistry meterRegistry, LongCounter requestCounter, LongCounter exceptionCounter, + LongCounter requestRangeNotSatisfiedExceptionCounter, LongCounter throttleCounter, LongCounter operationCounter, LongCounter unsuccessfulOperationCounter, @@ -28,6 +29,8 @@ public record RepositoriesMetrics( public static final String METRIC_REQUESTS_TOTAL = "es.repositories.requests.total"; public static final String METRIC_EXCEPTIONS_TOTAL = "es.repositories.exceptions.total"; + public static final String METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL = + "es.repositories.exceptions.request_range_not_satisfied.total"; public static final String METRIC_THROTTLES_TOTAL = "es.repositories.throttles.total"; public static final String METRIC_OPERATIONS_TOTAL = "es.repositories.operations.total"; public static final String METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL = "es.repositories.operations.unsuccessful.total"; @@ -40,6 +43,11 @@ public RepositoriesMetrics(MeterRegistry meterRegistry) { meterRegistry, meterRegistry.registerLongCounter(METRIC_REQUESTS_TOTAL, "repository request counter", "unit"), meterRegistry.registerLongCounter(METRIC_EXCEPTIONS_TOTAL, "repository request exception counter", "unit"), + meterRegistry.registerLongCounter( + METRIC_EXCEPTIONS_REQUEST_RANGE_NOT_SATISFIED_TOTAL, + "repository request RequestedRangeNotSatisfiedException counter", + "unit" + ), meterRegistry.registerLongCounter(METRIC_THROTTLES_TOTAL, "repository request throttle counter", "unit"), meterRegistry.registerLongCounter(METRIC_OPERATIONS_TOTAL, "repository operation counter", "unit"), meterRegistry.registerLongCounter(METRIC_UNSUCCESSFUL_OPERATIONS_TOTAL, "repository unsuccessful operation counter", "unit"), diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java index 2ac804b0597f8..470725017c937 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesModule.java @@ -8,6 +8,7 @@ package org.elasticsearch.repositories; +import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; @@ -20,7 +21,7 @@ import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.snapshots.SnapshotRestoreException; import org.elasticsearch.telemetry.TelemetryProvider; -import org.elasticsearch.transport.TransportService; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.NamedXContentRegistry; import java.util.ArrayList; @@ -40,7 +41,8 @@ public final class RepositoriesModule { public RepositoriesModule( Environment env, List repoPlugins, - TransportService transportService, + NodeClient client, + ThreadPool threadPool, ClusterService clusterService, BigArrays bigArrays, NamedXContentRegistry namedXContentRegistry, @@ -118,10 +120,10 @@ public RepositoriesModule( repositoriesService = new RepositoriesService( settings, clusterService, - transportService, repositoryTypes, internalRepositoryTypes, - transportService.getThreadPool(), + threadPool, + client, preRestoreChecks ); } diff --git a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java index 4884299bd1b4e..181fe6afb97d9 100644 --- a/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java +++ b/server/src/main/java/org/elasticsearch/repositories/RepositoriesService.java @@ -16,6 +16,7 @@ import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterState; @@ -45,10 +46,10 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.repositories.VerifyNodeRepositoryAction.Request; import org.elasticsearch.repositories.blobstore.MeteredBlobStoreRepository; import org.elasticsearch.snapshots.Snapshot; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import java.io.IOException; import java.util.ArrayList; @@ -103,8 +104,7 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C private final ClusterService clusterService; private final ThreadPool threadPool; - - private final VerifyNodeRepositoryAction verifyAction; + private final NodeClient client; private final Map internalRepositories = ConcurrentCollections.newConcurrentMap(); private volatile Map repositories = Collections.emptyMap(); @@ -116,16 +116,17 @@ public class RepositoriesService extends AbstractLifecycleComponent implements C public RepositoriesService( Settings settings, ClusterService clusterService, - TransportService transportService, Map typesRegistry, Map internalTypesRegistry, ThreadPool threadPool, + NodeClient client, List> preRestoreChecks ) { this.typesRegistry = typesRegistry; this.internalTypesRegistry = internalTypesRegistry; this.clusterService = clusterService; this.threadPool = threadPool; + this.client = client; // Doesn't make sense to maintain repositories on non-master and non-data nodes // Nothing happens there anyway if (DiscoveryNode.canContainData(settings) || DiscoveryNode.isMasterNode(settings)) { @@ -133,7 +134,6 @@ public RepositoriesService( clusterService.addHighPriorityApplier(this); } } - this.verifyAction = new VerifyNodeRepositoryAction(transportService, clusterService, this); this.repositoriesStatsArchive = new RepositoriesStatsArchive( REPOSITORIES_STATS_ARCHIVE_RETENTION_PERIOD.get(settings), REPOSITORIES_STATS_ARCHIVE_MAX_ARCHIVED_STATS.get(settings), @@ -412,6 +412,15 @@ public static void updateRepositoryUuidInMetadata( return; } + logger.info( + Strings.format( + "Registering repository [%s] with repository UUID [%s] and generation [%d]", + repositoryName, + repositoryData.getUuid(), + repositoryData.getGenId() + ) + ); + submitUnbatchedTask( clusterService, "update repository UUID [" + repositoryName + "] to [" + repositoryUuid + "]", @@ -528,11 +537,12 @@ protected void doRun() { final String verificationToken = repository.startVerification(); if (verificationToken != null) { try { - verifyAction.verify( - repositoryName, - verificationToken, + var nodeRequest = new Request(repositoryName, verificationToken); + client.execute( + VerifyNodeRepositoryCoordinationAction.TYPE, + nodeRequest, listener.delegateFailure( - (delegatedListener, verifyResponse) -> threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> { + (delegatedListener, response) -> threadPool.executor(ThreadPool.Names.SNAPSHOT).execute(() -> { try { repository.endVerification(verificationToken); } catch (Exception e) { @@ -540,7 +550,7 @@ protected void doRun() { delegatedListener.onFailure(e); return; } - delegatedListener.onResponse(verifyResponse); + delegatedListener.onResponse(response.nodes); }) ) ); diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java index 163450ea26e96..6750cced06191 100644 --- a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java +++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryAction.java @@ -8,149 +8,76 @@ package org.elasticsearch.repositories; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportChannel; -import org.elasticsearch.transport.TransportException; -import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequestHandler; -import org.elasticsearch.transport.TransportResponse; -import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicInteger; public class VerifyNodeRepositoryAction { - private static final Logger logger = LogManager.getLogger(VerifyNodeRepositoryAction.class); - public static final String ACTION_NAME = "internal:admin/repository/verify"; - - private final TransportService transportService; - - private final ClusterService clusterService; - - private final RepositoriesService repositoriesService; - - public VerifyNodeRepositoryAction( - TransportService transportService, - ClusterService clusterService, - RepositoriesService repositoriesService - ) { - this.transportService = transportService; - this.clusterService = clusterService; - this.repositoriesService = repositoriesService; - transportService.registerRequestHandler( - ACTION_NAME, - transportService.getThreadPool().executor(ThreadPool.Names.SNAPSHOT), - VerifyNodeRepositoryRequest::new, - new VerifyNodeRepositoryRequestHandler() - ); - } - - public void verify(String repository, String verificationToken, final ActionListener> listener) { - final DiscoveryNodes discoNodes = clusterService.state().nodes(); - final DiscoveryNode localNode = discoNodes.getLocalNode(); - - final Collection masterAndDataNodes = discoNodes.getMasterAndDataNodes().values(); - final List nodes = new ArrayList<>(); - for (DiscoveryNode node : masterAndDataNodes) { - if (RepositoriesService.isDedicatedVotingOnlyNode(node.getRoles()) == false) { - nodes.add(node); - } - } - final CopyOnWriteArrayList errors = new CopyOnWriteArrayList<>(); - final AtomicInteger counter = new AtomicInteger(nodes.size()); - for (final DiscoveryNode node : nodes) { - if (node.equals(localNode)) { - try { - doVerify(repository, verificationToken, localNode); - } catch (Exception e) { - logger.warn(() -> "[" + repository + "] failed to verify repository", e); - errors.add(new VerificationFailure(node.getId(), e)); - } - if (counter.decrementAndGet() == 0) { - finishVerification(repository, listener, nodes, errors); - } - } else { - transportService.sendRequest( - node, - ACTION_NAME, - new VerifyNodeRepositoryRequest(repository, verificationToken), - new TransportResponseHandler.Empty() { - @Override - public Executor executor() { - return TransportResponseHandler.TRANSPORT_WORKER; - } - - @Override - public void handleResponse() { - if (counter.decrementAndGet() == 0) { - finishVerification(repository, listener, nodes, errors); - } - } - - @Override - public void handleException(TransportException exp) { - errors.add(new VerificationFailure(node.getId(), exp)); - if (counter.decrementAndGet() == 0) { - finishVerification(repository, listener, nodes, errors); - } - } - } - ); - } + public static final ActionType TYPE = new ActionType<>(ACTION_NAME); + + // no construction + private VerifyNodeRepositoryAction() {} + + public static class TransportAction extends HandledTransportAction { + + private final ClusterService clusterService; + private final RepositoriesService repositoriesService; + + @Inject + public TransportAction( + TransportService transportService, + ActionFilters actionFilters, + ThreadPool threadPool, + ClusterService clusterService, + RepositoriesService repositoriesService + ) { + super(ACTION_NAME, transportService, actionFilters, Request::new, threadPool.executor(ThreadPool.Names.SNAPSHOT)); + this.clusterService = clusterService; + this.repositoriesService = repositoriesService; } - } - private static void finishVerification( - String repositoryName, - ActionListener> listener, - List nodes, - CopyOnWriteArrayList errors - ) { - if (errors.isEmpty() == false) { - RepositoryVerificationException e = new RepositoryVerificationException(repositoryName, errors.toString()); - for (VerificationFailure error : errors) { - e.addSuppressed(error.getCause()); + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + DiscoveryNode localNode = clusterService.state().nodes().getLocalNode(); + try { + Repository repository = repositoriesService.repository(request.repository); + repository.verify(request.verificationToken, localNode); + listener.onResponse(ActionResponse.Empty.INSTANCE); + } catch (Exception e) { + logger.warn(() -> "[" + request.repository + "] failed to verify repository", e); + listener.onFailure(e); } - listener.onFailure(e); - } else { - listener.onResponse(nodes); } } - private void doVerify(String repositoryName, String verificationToken, DiscoveryNode localNode) { - Repository repository = repositoriesService.repository(repositoryName); - repository.verify(verificationToken, localNode); - } - - public static class VerifyNodeRepositoryRequest extends TransportRequest { + public static class Request extends ActionRequest { - private final String repository; - private final String verificationToken; + protected final String repository; + protected final String verificationToken; - public VerifyNodeRepositoryRequest(StreamInput in) throws IOException { + public Request(StreamInput in) throws IOException { super(in); repository = in.readString(); verificationToken = in.readString(); } - VerifyNodeRepositoryRequest(String repository, String verificationToken) { + Request(String repository, String verificationToken) { this.repository = repository; this.verificationToken = verificationToken; } @@ -161,19 +88,10 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(repository); out.writeString(verificationToken); } - } - class VerifyNodeRepositoryRequestHandler implements TransportRequestHandler { @Override - public void messageReceived(VerifyNodeRepositoryRequest request, TransportChannel channel, Task task) throws Exception { - DiscoveryNode localNode = clusterService.state().nodes().getLocalNode(); - try { - doVerify(request.repository, request.verificationToken, localNode); - } catch (Exception ex) { - logger.warn(() -> "[" + request.repository + "] failed to verify repository", ex); - throw ex; - } - channel.sendResponse(TransportResponse.Empty.INSTANCE); + public ActionRequestValidationException validate() { + return null; } } diff --git a/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java new file mode 100644 index 0000000000000..b892ff93c7a9c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/repositories/VerifyNodeRepositoryCoordinationAction.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.repositories; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.repositories.VerifyNodeRepositoryAction.Request; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportResponseHandler; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicInteger; + +public class VerifyNodeRepositoryCoordinationAction { + + public static final String NAME = "internal:admin/repository/verify/coordinate"; + public static final ActionType TYPE = new ActionType<>(NAME); + + private VerifyNodeRepositoryCoordinationAction() {} + + public static class Response extends ActionResponse { + + final List nodes; + + public Response(List nodes) { + this.nodes = nodes; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + TransportAction.localOnly(); + } + } + + public static class LocalAction extends TransportAction { + + private final TransportService transportService; + private final ClusterService clusterService; + private final NodeClient client; + + @Inject + public LocalAction( + ActionFilters actionFilters, + TransportService transportService, + ClusterService clusterService, + NodeClient client + ) { + super(NAME, actionFilters, transportService.getTaskManager()); + this.transportService = transportService; + this.clusterService = clusterService; + this.client = client; + } + + @Override + protected void doExecute(Task task, Request request, ActionListener listener) { + final DiscoveryNodes discoNodes = clusterService.state().nodes(); + final DiscoveryNode localNode = discoNodes.getLocalNode(); + + final Collection masterAndDataNodes = discoNodes.getMasterAndDataNodes().values(); + final List nodes = new ArrayList<>(); + for (DiscoveryNode node : masterAndDataNodes) { + if (RepositoriesService.isDedicatedVotingOnlyNode(node.getRoles()) == false) { + nodes.add(node); + } + } + final CopyOnWriteArrayList errors = new CopyOnWriteArrayList<>(); + final AtomicInteger counter = new AtomicInteger(nodes.size()); + for (final DiscoveryNode node : nodes) { + transportService.sendRequest( + node, + VerifyNodeRepositoryAction.ACTION_NAME, + request, + new TransportResponseHandler() { + + @Override + public ActionResponse.Empty read(StreamInput in) throws IOException { + return ActionResponse.Empty.INSTANCE; + } + + @Override + public Executor executor() { + return TransportResponseHandler.TRANSPORT_WORKER; + } + + @Override + public void handleResponse(ActionResponse.Empty _ignore) { + if (counter.decrementAndGet() == 0) { + finishVerification(request.repository, listener, nodes, errors); + } + } + + @Override + public void handleException(TransportException exp) { + errors.add(new VerificationFailure(node.getId(), exp)); + if (counter.decrementAndGet() == 0) { + finishVerification(request.repository, listener, nodes, errors); + } + } + } + ); + } + } + + private static void finishVerification( + String repositoryName, + ActionListener listener, + List nodes, + CopyOnWriteArrayList errors + ) { + if (errors.isEmpty() == false) { + RepositoryVerificationException e = new RepositoryVerificationException(repositoryName, errors.toString()); + for (VerificationFailure error : errors) { + e.addSuppressed(error.getCause()); + } + listener.onFailure(e); + } else { + listener.onResponse(new Response(nodes)); + } + } + } +} diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index dac5ab97f2962..e27ba56bed974 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -2709,6 +2709,22 @@ public void onFailure(Exception e) { }, true); maybeWriteIndexLatest(newGen); + if (filteredRepositoryData.getUuid().equals(RepositoryData.MISSING_UUID) && SnapshotsService.includesUUIDs(version)) { + assert newRepositoryData.getUuid().equals(RepositoryData.MISSING_UUID) == false; + logger.info( + Strings.format( + "Generated new repository UUID [%s] for repository [%s] in generation [%d]", + newRepositoryData.getUuid(), + metadata.name(), + newGen + ) + ); + } else { + // repo UUID is not new + assert filteredRepositoryData.getUuid().equals(newRepositoryData.getUuid()) + : filteredRepositoryData.getUuid() + " vs " + newRepositoryData.getUuid(); + } + // Step 3: Update CS to reflect new repository generation. final String setSafeGenerationSource = "set safe repository generation [" + metadata.name() + "][" + newGen + "]"; submitUnbatchedTask(setSafeGenerationSource, new ClusterStateUpdateTask() { diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java index d5ed840edb2b5..026d8ba26b118 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStatsAction.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestUtils.getTimeout; @@ -26,6 +27,8 @@ @ServerlessScope(Scope.INTERNAL) public class RestClusterStatsAction extends BaseRestHandler { + private static final Set SUPPORTED_CAPABILITIES = Set.of("human-readable-total-docs-size"); + @Override public List routes() { return List.of(new Route(GET, "/_cluster/stats"), new Route(GET, "/_cluster/stats/nodes/{nodeId}")); @@ -38,7 +41,7 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { - ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest().nodesIds(request.paramAsStringArray("nodeId", null)); + ClusterStatsRequest clusterStatsRequest = new ClusterStatsRequest(request.paramAsStringArray("nodeId", null)); clusterStatsRequest.timeout(getTimeout(request)); return channel -> new RestCancellableNodeClient(client, request.getHttpChannel()).admin() .cluster() @@ -49,4 +52,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC public boolean canTripCircuitBreaker() { return false; } + + @Override + public Set supportedCapabilities() { + return SUPPORTED_CAPABILITIES; + } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java index ad7bdc8a2c9b0..37870c44fe256 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestDeleteSnapshotAction.java @@ -11,14 +11,17 @@ import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestUtils; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; import java.io.IOException; import java.util.List; +import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.DELETE; import static org.elasticsearch.rest.RestUtils.getMasterNodeTimeout; @@ -29,6 +32,11 @@ @ServerlessScope(Scope.INTERNAL) public class RestDeleteSnapshotAction extends BaseRestHandler { + private static final Set SUPPORTED_QUERY_PARAMETERS = Set.of(RestUtils.REST_MASTER_TIMEOUT_PARAM, "wait_for_completion"); + private static final Set ALL_SUPPORTED_PARAMETERS = Set.copyOf( + Sets.union(SUPPORTED_QUERY_PARAMETERS, Set.of("repository", "snapshot")) + ); + @Override public List routes() { return List.of(new Route(DELETE, "/_snapshot/{repository}/{snapshot}")); @@ -39,12 +47,23 @@ public String getName() { return "delete_snapshot_action"; } + @Override + public Set allSupportedParameters() { + return ALL_SUPPORTED_PARAMETERS; + } + + @Override + public Set supportedQueryParameters() { + return SUPPORTED_QUERY_PARAMETERS; + } + @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { String repository = request.param("repository"); String[] snapshots = Strings.splitStringByCommaToArray(request.param("snapshot")); DeleteSnapshotRequest deleteSnapshotRequest = new DeleteSnapshotRequest(repository, snapshots); deleteSnapshotRequest.masterNodeTimeout(getMasterNodeTimeout(request)); + deleteSnapshotRequest.waitForCompletion(request.paramAsBoolean("wait_for_completion", deleteSnapshotRequest.waitForCompletion())); return channel -> client.admin().cluster().deleteSnapshot(deleteSnapshotRequest, new RestToXContentListener<>(channel)); } } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java index 9cf2d6a2ed395..6fbb028db7f37 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestNodesHotThreadsAction.java @@ -103,15 +103,17 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { String[] nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")); - NodesHotThreadsRequest nodesHotThreadsRequest = new NodesHotThreadsRequest(nodesIds); - nodesHotThreadsRequest.threads(request.paramAsInt("threads", nodesHotThreadsRequest.threads())); - nodesHotThreadsRequest.ignoreIdleThreads(request.paramAsBoolean("ignore_idle_threads", nodesHotThreadsRequest.ignoreIdleThreads())); - nodesHotThreadsRequest.type(HotThreads.ReportType.of(request.param("type", nodesHotThreadsRequest.type().getTypeValue()))); - nodesHotThreadsRequest.sortOrder( - HotThreads.SortOrder.of(request.param("sort", nodesHotThreadsRequest.sortOrder().getOrderValue())) + NodesHotThreadsRequest nodesHotThreadsRequest = new NodesHotThreadsRequest( + nodesIds, + new HotThreads.RequestOptions( + request.paramAsInt("threads", HotThreads.RequestOptions.DEFAULT.threads()), + HotThreads.ReportType.of(request.param("type", HotThreads.RequestOptions.DEFAULT.reportType().getTypeValue())), + HotThreads.SortOrder.of(request.param("sort", HotThreads.RequestOptions.DEFAULT.sortOrder().getOrderValue())), + request.paramAsTime("interval", HotThreads.RequestOptions.DEFAULT.interval()), + request.paramAsInt("snapshots", HotThreads.RequestOptions.DEFAULT.snapshots()), + request.paramAsBoolean("ignore_idle_threads", HotThreads.RequestOptions.DEFAULT.ignoreIdleThreads()) + ) ); - nodesHotThreadsRequest.interval(request.paramAsTime("interval", nodesHotThreadsRequest.interval())); - nodesHotThreadsRequest.snapshots(request.paramAsInt("snapshots", nodesHotThreadsRequest.snapshots())); nodesHotThreadsRequest.timeout(getTimeout(request)); return channel -> client.execute(TransportNodesHotThreadsAction.TYPE, nodesHotThreadsRequest, new RestResponseListener<>(channel) { @Override diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java index 2956bd930f4fa..3d4af0c2c2fd0 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestReloadSecureSettingsAction.java @@ -62,8 +62,9 @@ public List routes() { @Override public RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - final NodesReloadSecureSettingsRequest reloadSecureSettingsRequest = new NodesReloadSecureSettingsRequest(); - reloadSecureSettingsRequest.nodesIds(Strings.splitStringByCommaToArray(request.param("nodeId"))); + final NodesReloadSecureSettingsRequest reloadSecureSettingsRequest = new NodesReloadSecureSettingsRequest( + Strings.splitStringByCommaToArray(request.param("nodeId")) + ); reloadSecureSettingsRequest.timeout(getTimeout(request)); request.withContentOrSourceParamParserOrNull(parser -> { if (parser != null) { diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java index ca3bcfbcd38e0..edf8f12b69579 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java @@ -514,6 +514,12 @@ protected Table getTableWithHeader(final RestRequest request) { ); table.addCell("pri.dense_vector.value_count", "default:false;text-align:right;desc:total count of indexed dense vector"); + table.addCell( + "sparse_vector.value_count", + "sibling:pri;alias:svc,sparseVectorCount;default:false;text-align:right;desc:total count of indexed sparse vectors" + ); + table.addCell("pri.sparse_vector.value_count", "default:false;text-align:right;desc:total count of indexed sparse vectors"); + table.endHeaders(); return table; } @@ -791,6 +797,9 @@ Table buildTable( table.addCell(totalStats.getDenseVectorStats() == null ? null : totalStats.getDenseVectorStats().getValueCount()); table.addCell(primaryStats.getDenseVectorStats() == null ? null : primaryStats.getDenseVectorStats().getValueCount()); + table.addCell(totalStats.getSparseVectorStats() == null ? null : totalStats.getSparseVectorStats().getValueCount()); + table.addCell(primaryStats.getSparseVectorStats() == null ? null : primaryStats.getSparseVectorStats().getValueCount()); + table.endRow(); }); diff --git a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java index d9a34fe36c860..fffa272d8fd12 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/cat/RestShardsAction.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.DenseVectorStats; import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.rest.RestRequest; @@ -253,7 +254,11 @@ protected Table getTableWithHeader(final RestRequest request) { ); table.addCell( "dense_vector.value_count", - "alias:dvc,denseVectorCount;default:false;text-align:right;desc:total count of indexed dense vector" + "alias:dvc,denseVectorCount;default:false;text-align:right;desc:number of indexed dense vectors in shard" + ); + table.addCell( + "sparse_vector.value_count", + "alias:svc,sparseVectorCount;default:false;text-align:right;desc:number of indexed sparse vectors in shard" ); table.endHeaders(); @@ -420,6 +425,7 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe table.addCell(getOrNull(commonStats, CommonStats::getBulk, BulkStats::getAvgSizeInBytes)); table.addCell(getOrNull(commonStats, CommonStats::getDenseVectorStats, DenseVectorStats::getValueCount)); + table.addCell(getOrNull(commonStats, CommonStats::getSparseVectorStats, SparseVectorStats::getValueCount)); table.endRow(); } diff --git a/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java b/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java new file mode 100644 index 0000000000000..d4d78bf08844b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/script/ScriptFeatures.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.script; + +import org.elasticsearch.features.FeatureSpecification; +import org.elasticsearch.features.NodeFeature; + +import java.util.Set; + +public final class ScriptFeatures implements FeatureSpecification { + @Override + public Set getFeatures() { + return Set.of(VectorScoreScriptUtils.HAMMING_DISTANCE_FUNCTION); + } +} diff --git a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java index b071739321eaf..bccdd5782f277 100644 --- a/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java +++ b/server/src/main/java/org/elasticsearch/script/VectorScoreScriptUtils.java @@ -9,6 +9,8 @@ package org.elasticsearch.script; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.features.NodeFeature; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.script.field.vectors.DenseVector; import org.elasticsearch.script.field.vectors.DenseVectorDocValuesField; @@ -18,6 +20,8 @@ public class VectorScoreScriptUtils { + public static final NodeFeature HAMMING_DISTANCE_FUNCTION = new NodeFeature("script.hamming"); + public static class DenseVectorFunction { protected final ScoreScript scoreScript; protected final DenseVectorDocValuesField field; @@ -187,6 +191,52 @@ public double l1norm() { } } + // Calculate Hamming distances between a query's dense vector and documents' dense vectors + public interface HammingDistanceInterface { + int hamming(); + } + + public static class ByteHammingDistance extends ByteDenseVectorFunction implements HammingDistanceInterface { + + public ByteHammingDistance(ScoreScript scoreScript, DenseVectorDocValuesField field, List queryVector) { + super(scoreScript, field, queryVector); + } + + public ByteHammingDistance(ScoreScript scoreScript, DenseVectorDocValuesField field, byte[] queryVector) { + super(scoreScript, field, queryVector); + } + + public int hamming() { + setNextVector(); + return field.get().hamming(queryVector); + } + } + + public static final class Hamming { + + private final HammingDistanceInterface function; + + @SuppressWarnings("unchecked") + public Hamming(ScoreScript scoreScript, Object queryVector, String fieldName) { + DenseVectorDocValuesField field = (DenseVectorDocValuesField) scoreScript.field(fieldName); + if (field.getElementType() != DenseVectorFieldMapper.ElementType.BYTE) { + throw new IllegalArgumentException("hamming distance is only supported for byte vectors"); + } + if (queryVector instanceof List) { + function = new ByteHammingDistance(scoreScript, field, (List) queryVector); + } else if (queryVector instanceof String s) { + byte[] parsedQueryVector = HexFormat.of().parseHex(s); + function = new ByteHammingDistance(scoreScript, field, parsedQueryVector); + } else { + throw new IllegalArgumentException("Unsupported input object for byte vectors: " + queryVector.getClass().getName()); + } + } + + public double hamming() { + return function.hamming(); + } + } + // Calculate l2 norm (Manhattan distance) between a query's dense vector and documents' dense vectors public interface L2NormInterface { double l2norm(); diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/BinaryDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/BinaryDenseVector.java index cffddfabf4aba..4fbfdcf9771a3 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/BinaryDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/BinaryDenseVector.java @@ -83,6 +83,16 @@ public double l1Norm(List queryVector) { return l1norm; } + @Override + public int hamming(byte[] queryVector) { + throw new UnsupportedOperationException("hamming distance is not supported for float vectors"); + } + + @Override + public int hamming(List queryVector) { + throw new UnsupportedOperationException("hamming distance is not supported for float vectors"); + } + @Override public double l2Norm(byte[] queryVector) { throw new UnsupportedOperationException("use [double l2Norm(float[] queryVector)] instead"); diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java index a986b62ce8496..c009397452c8a 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteBinaryDenseVector.java @@ -100,6 +100,20 @@ public double l1Norm(List queryVector) { return result; } + @Override + public int hamming(byte[] queryVector) { + return VectorUtil.xorBitCount(queryVector, vectorValue); + } + + @Override + public int hamming(List queryVector) { + int distance = 0; + for (int i = 0; i < queryVector.size(); i++) { + distance += Integer.bitCount((queryVector.get(i).intValue() ^ vectorValue[i]) & 0xFF); + } + return distance; + } + @Override public double l2Norm(byte[] queryVector) { return Math.sqrt(VectorUtil.squareDistance(queryVector, vectorValue)); diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java index b00b6703872ab..e0ba032826aa1 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/ByteKnnDenseVector.java @@ -101,6 +101,20 @@ public double l1Norm(List queryVector) { return result; } + @Override + public int hamming(byte[] queryVector) { + return VectorUtil.xorBitCount(queryVector, docVector); + } + + @Override + public int hamming(List queryVector) { + int distance = 0; + for (int i = 0; i < queryVector.size(); i++) { + distance += Integer.bitCount((queryVector.get(i).intValue() ^ docVector[i]) & 0xFF); + } + return distance; + } + @Override public double l2Norm(byte[] queryVector) { return Math.sqrt(VectorUtil.squareDistance(docVector, queryVector)); diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java index d18ae16746819..a768e8add6663 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/DenseVector.java @@ -14,8 +14,7 @@ /** * DenseVector value type for the painless. - */ -/* dotProduct, l1Norm, l2Norm, cosineSimilarity have three flavors depending on the type of the queryVector + * dotProduct, l1Norm, l2Norm, cosineSimilarity have three flavors depending on the type of the queryVector * 1) float[], this is for the ScoreScriptUtils class bindings which have converted a List based query vector into an array * 2) List, A painless script will typically use Lists since they are easy to pass as params and have an easy * literal syntax. Working with Lists directly, instead of converting to a float[], trades off runtime operations against @@ -74,6 +73,24 @@ default double l1Norm(Object queryVector) { throw new IllegalArgumentException(badQueryVectorType(queryVector)); } + int hamming(byte[] queryVector); + + int hamming(List queryVector); + + @SuppressWarnings("unchecked") + default int hamming(Object queryVector) { + if (queryVector instanceof List list) { + checkDimensions(getDims(), list.size()); + return hamming((List) list); + } + if (queryVector instanceof byte[] bytes) { + checkDimensions(getDims(), bytes.length); + return hamming(bytes); + } + + throw new IllegalArgumentException(badQueryVectorType(queryVector)); + } + double l2Norm(byte[] queryVector); double l2Norm(float[] queryVector); @@ -231,6 +248,16 @@ public double l1Norm(List queryVector) { throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); } + @Override + public int hamming(byte[] queryVector) { + throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); + } + + @Override + public int hamming(List queryVector) { + throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); + } + @Override public double l2Norm(byte[] queryVector) { throw new IllegalArgumentException(MISSING_VECTOR_FIELD_MESSAGE); diff --git a/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVector.java b/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVector.java index 1605f179e36aa..7f94f029dcbb3 100644 --- a/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVector.java +++ b/server/src/main/java/org/elasticsearch/script/field/vectors/KnnDenseVector.java @@ -85,6 +85,16 @@ public double l1Norm(List queryVector) { return result; } + @Override + public int hamming(byte[] queryVector) { + throw new UnsupportedOperationException("hamming distance is not supported for float vectors"); + } + + @Override + public int hamming(List queryVector) { + throw new UnsupportedOperationException("hamming distance is not supported for float vectors"); + } + @Override public double l2Norm(byte[] queryVector) { throw new UnsupportedOperationException("use [double l2Norm(float[] queryVector)] instead"); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index acdb24b9109af..26204e1a2530f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -380,7 +380,7 @@ private void mapSegmentCountsToGlobalCounts(LongUnaryOperator mapping) throws IO for (long i = 1; i < segmentDocCounts.size(); i++) { // We use set(...) here, because we need to reset the slow to 0. // segmentDocCounts get reused over the segments and otherwise counts would be too high. - long inc = segmentDocCounts.set(i, 0); + long inc = segmentDocCounts.getAndSet(i, 0); if (inc == 0) { continue; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java index df120f608eb97..4c1477f532648 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregationBuilder.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.ToLongFunction; import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; @@ -192,11 +193,11 @@ protected ScriptedMetricAggregatorFactory doBuild(AggregationContext context, Ag validateScript(REDUCE_SCRIPT_FIELD.getPreferredName(), name, reduceScript, settings); if (combineScript == null) { - throw new IllegalArgumentException("[combineScript] must not be null: [" + name + "]"); + throw new IllegalArgumentException("[" + COMBINE_SCRIPT_FIELD.getPreferredName() + "] must not be null: [" + name + "]"); } if (reduceScript == null) { - throw new IllegalArgumentException("[reduceScript] must not be null: [" + name + "]"); + throw new IllegalArgumentException("[" + REDUCE_SCRIPT_FIELD.getPreferredName() + "] must not be null: [" + name + "]"); } // Extract params from scripts and pass them along to ScriptedMetricAggregatorFactory, since it won't have @@ -292,6 +293,11 @@ public TransportVersion getMinimalSupportedVersion() { return TransportVersions.ZERO; } + @Override + public boolean supportsParallelCollection(ToLongFunction fieldCardinalityResolver) { + return false; + } + @Override public int hashCode() { return Objects.hash(super.hashCode(), initScript, mapScript, combineScript, reduceScript, params); diff --git a/server/src/main/java/org/elasticsearch/snapshots/InternalSnapshotsInfoService.java b/server/src/main/java/org/elasticsearch/snapshots/InternalSnapshotsInfoService.java index da0b0d134b0f8..cc376bc6c79c4 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/InternalSnapshotsInfoService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/InternalSnapshotsInfoService.java @@ -59,7 +59,7 @@ public final class InternalSnapshotsInfoService implements ClusterStateListener, ); private final ThreadPool threadPool; - private final Supplier repositoriesService; + private final RepositoriesService repositoriesService; private final Supplier rerouteService; /** contains the snapshot shards for which the size is known **/ @@ -87,11 +87,11 @@ public final class InternalSnapshotsInfoService implements ClusterStateListener, public InternalSnapshotsInfoService( final Settings settings, final ClusterService clusterService, - final Supplier repositoriesServiceSupplier, + final RepositoriesService repositoriesService, final Supplier rerouteServiceSupplier ) { this.threadPool = clusterService.getClusterApplierService().threadPool(); - this.repositoriesService = repositoriesServiceSupplier; + this.repositoriesService = repositoriesService; this.rerouteService = rerouteServiceSupplier; this.knownSnapshotShards = ImmutableOpenMap.of(); this.unknownSnapshotShards = new LinkedHashSet<>(); @@ -210,9 +210,7 @@ private class FetchingSnapshotShardSizeRunnable extends AbstractRunnable { @Override protected void doRun() throws Exception { - final RepositoriesService repositories = repositoriesService.get(); - assert repositories != null; - final Repository repository = repositories.repository(snapshotShard.snapshot.getRepository()); + final Repository repository = repositoriesService.repository(snapshotShard.snapshot.getRepository()); logger.debug("fetching snapshot shard size for {}", snapshotShard); final long snapshotShardSize = repository.getShardSnapshotStatus( diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java index 7ca92ebfdcf32..cd7516a8f1232 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotsService.java @@ -1996,8 +1996,11 @@ private void failSnapshotCompletionListeners(Snapshot snapshot, Exception e, Con /** * Deletes snapshots from the repository. In-progress snapshots matched by the delete will be aborted before deleting them. * + * When wait_for_completion is set to true, the passed action listener will only complete when all + * matching snapshots are deleted, when it is false it will complete as soon as the deletes are scheduled + * * @param request delete snapshot request - * @param listener listener + * @param listener listener a listener which will be resolved according to the wait_for_completion parameter */ public void deleteSnapshots(final DeleteSnapshotRequest request, final ActionListener listener) { final String repositoryName = request.repository(); @@ -2190,10 +2193,12 @@ public void clusterStateProcessed(ClusterState oldState, ClusterState newState) Runnable::run ); } - if (newDelete == null) { + if (newDelete == null || request.waitForCompletion() == false) { listener.onResponse(null); } else { addDeleteListener(newDelete.uuid(), listener); + } + if (newDelete != null) { if (reusedExistingDelete) { return; } diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java index 7d8b966451d37..db80dde4c226b 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskCancellationService.java @@ -212,7 +212,7 @@ public void handleResponse() { @Override public void handleException(TransportException exp) { final Throwable cause = ExceptionsHelper.unwrapCause(exp); - assert cause instanceof ElasticsearchSecurityException == false; + assert cause instanceof ElasticsearchSecurityException == false : new AssertionError(exp); if (isUnimportantBanFailure(cause)) { logger.debug( () -> format("cannot send ban for tasks with the parent [%s] on connection [%s]", taskId, connection), @@ -261,7 +261,7 @@ public void handleResponse() {} @Override public void handleException(TransportException exp) { final Throwable cause = ExceptionsHelper.unwrapCause(exp); - assert cause instanceof ElasticsearchSecurityException == false; + assert cause instanceof ElasticsearchSecurityException == false : new AssertionError(exp); if (isUnimportantBanFailure(cause)) { logger.debug( () -> format( diff --git a/server/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java b/server/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java index 29a7d5df08b7b..0b1026dfbfa6b 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ScalingExecutorBuilder.java @@ -35,6 +35,7 @@ public final class ScalingExecutorBuilder extends ExecutorBuilder maxSetting; private final Setting keepAliveSetting; private final boolean rejectAfterShutdown; + private final EsExecutors.TaskTrackingConfig trackingConfig; /** * Construct a scaling executor builder; the settings will have the @@ -76,12 +77,38 @@ public ScalingExecutorBuilder( final TimeValue keepAlive, final boolean rejectAfterShutdown, final String prefix + ) { + this(name, core, max, keepAlive, rejectAfterShutdown, prefix, EsExecutors.TaskTrackingConfig.DO_NOT_TRACK); + } + + /** + * Construct a scaling executor builder; the settings will have the + * specified key prefix. + * + * @param name the name of the executor + * @param core the minimum number of threads in the pool + * @param max the maximum number of threads in the pool + * @param keepAlive the time that spare threads above {@code core} + * threads will be kept alive + * @param prefix the prefix for the settings keys + * @param rejectAfterShutdown set to {@code true} if the executor should reject tasks after shutdown + * @param trackingConfig configuration that'll indicate if we should track statistics about task execution time + */ + public ScalingExecutorBuilder( + final String name, + final int core, + final int max, + final TimeValue keepAlive, + final boolean rejectAfterShutdown, + final String prefix, + final EsExecutors.TaskTrackingConfig trackingConfig ) { super(name); this.coreSetting = Setting.intSetting(settingsKey(prefix, "core"), core, Setting.Property.NodeScope); this.maxSetting = Setting.intSetting(settingsKey(prefix, "max"), max, Setting.Property.NodeScope); this.keepAliveSetting = Setting.timeSetting(settingsKey(prefix, "keep_alive"), keepAlive, Setting.Property.NodeScope); this.rejectAfterShutdown = rejectAfterShutdown; + this.trackingConfig = trackingConfig; } @Override @@ -104,7 +131,8 @@ ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final Th int max = settings.max; final ThreadPool.Info info = new ThreadPool.Info(name(), ThreadPool.ThreadPoolType.SCALING, core, max, keepAlive, null); final ThreadFactory threadFactory = EsExecutors.daemonThreadFactory(EsExecutors.threadName(settings.nodeName, name())); - final ExecutorService executor = EsExecutors.newScaling( + ExecutorService executor; + executor = EsExecutors.newScaling( settings.nodeName + "/" + name(), core, max, @@ -112,7 +140,8 @@ ThreadPool.ExecutorHolder build(final ScalingExecutorSettings settings, final Th TimeUnit.MILLISECONDS, rejectAfterShutdown, threadFactory, - threadContext + threadContext, + trackingConfig ); return new ThreadPool.ExecutorHolder(executor, info); } diff --git a/server/src/main/java/org/elasticsearch/transport/TransportRequest.java b/server/src/main/java/org/elasticsearch/transport/TransportRequest.java index 7646703faaa70..937344969ce44 100644 --- a/server/src/main/java/org/elasticsearch/transport/TransportRequest.java +++ b/server/src/main/java/org/elasticsearch/transport/TransportRequest.java @@ -16,16 +16,6 @@ import java.io.IOException; public abstract class TransportRequest extends TransportMessage implements TaskAwareRequest { - public static class Empty extends TransportRequest { - public static final Empty INSTANCE = new Empty(); - - public Empty() {} - - public Empty(StreamInput in) throws IOException { - super(in); - } - } - /** * Parent of this request. Defaults to {@link TaskId#EMPTY_TASK_ID}, meaning "no parent". */ diff --git a/server/src/main/java/org/elasticsearch/watcher/FileWatcher.java b/server/src/main/java/org/elasticsearch/watcher/FileWatcher.java index 3f9cd42504cd5..5fd9a46d6984f 100644 --- a/server/src/main/java/org/elasticsearch/watcher/FileWatcher.java +++ b/server/src/main/java/org/elasticsearch/watcher/FileWatcher.java @@ -11,15 +11,17 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.common.Strings; import org.elasticsearch.common.hash.MessageDigests; -import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.util.CollectionUtils; import java.io.IOException; +import java.io.InputStream; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; import java.security.AccessControlException; import java.util.Arrays; +import java.util.stream.StreamSupport; /** * File resources watcher @@ -114,6 +116,22 @@ void onDirectoryDeleted() {} void onFileDeleted() {} } + protected boolean fileExists(Path path) { + return Files.exists(path); + } + + protected BasicFileAttributes readAttributes(Path path) throws IOException { + return Files.readAttributes(path, BasicFileAttributes.class); + } + + protected InputStream newInputStream(Path path) throws IOException { + return Files.newInputStream(path); + } + + protected DirectoryStream listFiles(Path path) throws IOException { + return Files.newDirectoryStream(path); + } + private class FileObserver extends Observer { private long length; private long lastModified; @@ -131,10 +149,10 @@ public void checkAndNotify() throws IOException { long prevLastModified = lastModified; byte[] prevDigest = digest; - exists = Files.exists(path); + exists = fileExists(path); // TODO we might use the new NIO2 API to get real notification? if (exists) { - BasicFileAttributes attributes = Files.readAttributes(path, BasicFileAttributes.class); + BasicFileAttributes attributes = readAttributes(path); isDirectory = attributes.isDirectory(); if (isDirectory) { length = 0; @@ -202,7 +220,7 @@ public void checkAndNotify() throws IOException { } private byte[] calculateDigest() { - try (var in = Files.newInputStream(path)) { + try (var in = newInputStream(path)) { return MessageDigests.digest(in, MessageDigests.md5()); } catch (IOException e) { logger.warn( @@ -215,9 +233,9 @@ private byte[] calculateDigest() { } private void init(boolean initial) throws IOException { - exists = Files.exists(path); + exists = fileExists(path); if (exists) { - BasicFileAttributes attributes = Files.readAttributes(path, BasicFileAttributes.class); + BasicFileAttributes attributes = readAttributes(path); isDirectory = attributes.isDirectory(); if (isDirectory) { onDirectoryCreated(initial); @@ -245,9 +263,9 @@ private Observer createChild(Path file, boolean initial) throws IOException { } private Path[] listFiles() throws IOException { - final Path[] files = FileSystemUtils.files(path); - Arrays.sort(files); - return files; + try (var dirs = FileWatcher.this.listFiles(path)) { + return StreamSupport.stream(dirs.spliterator(), false).sorted().toArray(Path[]::new); + } } private Observer[] listChildren(boolean initial) throws IOException { diff --git a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification index d8a29a84ddbb7..5192ea2b4b108 100644 --- a/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification +++ b/server/src/main/resources/META-INF/services/org.elasticsearch.features.FeatureSpecification @@ -15,4 +15,5 @@ org.elasticsearch.indices.IndicesFeatures org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures org.elasticsearch.index.mapper.MapperFeatures org.elasticsearch.search.retriever.RetrieversFeatures +org.elasticsearch.script.ScriptFeatures org.elasticsearch.reservedstate.service.FileSettingsFeatures diff --git a/server/src/main/resources/org/elasticsearch/TransportVersions.csv b/server/src/main/resources/org/elasticsearch/TransportVersions.csv index ef0c641bed04f..ba1dab5589ee2 100644 --- a/server/src/main/resources/org/elasticsearch/TransportVersions.csv +++ b/server/src/main/resources/org/elasticsearch/TransportVersions.csv @@ -69,6 +69,7 @@ 7.17.19,7171999 7.17.20,7172099 7.17.21,7172199 +7.17.22,7172299 8.0.0,8000099 8.0.1,8000199 8.1.0,8010099 @@ -121,3 +122,4 @@ 8.13.3,8595000 8.13.4,8595001 8.14.0,8636001 +8.14.1,8636001 diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 52a2db62ac903..681a52eb84b8a 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -83,7 +83,7 @@ grant codeBase "${codebase.elasticsearch-preallocate}" { permission java.lang.reflect.ReflectPermission "newProxyInPackage.org.elasticsearch.preallocate"; }; -grant codeBase "${codebase.elasticsearch-vec}" { +grant codeBase "${codebase.elasticsearch-simdvec}" { // for access MemorySegmentIndexInput internals permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; diff --git a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv index 73f60f2e5ea7e..b7ca55a2b2b0d 100644 --- a/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv +++ b/server/src/main/resources/org/elasticsearch/index/IndexVersions.csv @@ -69,6 +69,7 @@ 7.17.19,7171999 7.17.20,7172099 7.17.21,7172199 +7.17.22,7172299 8.0.0,8000099 8.0.1,8000199 8.1.0,8010099 @@ -121,3 +122,4 @@ 8.13.3,8503000 8.13.4,8503000 8.14.0,8505000 +8.14.1,8505000 diff --git a/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java b/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java index 3bdf5814878a7..0543bce08a4f0 100644 --- a/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java +++ b/server/src/test/java/org/elasticsearch/action/ActionListenerTests.java @@ -10,6 +10,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.core.Assertions; import org.elasticsearch.core.CheckedConsumer; @@ -370,6 +371,52 @@ public void onFailure(Exception e) { assertThat(exReference.get(), instanceOf(IllegalArgumentException.class)); } + public void testAssertAtLeastOnceWillLogAssertionErrorWhenNotResolved() throws Exception { + assumeTrue("assertAtLeastOnce will be a no-op when assertions are disabled", Assertions.ENABLED); + ActionListener listenerRef = ActionListener.assertAtLeastOnce(ActionListener.running(() -> { + // Do nothing, but don't use ActionListener.noop() as it'll never be garbage collected + })); + // Nullify reference so it becomes unreachable + listenerRef = null; + assertBusy(() -> { + System.gc(); + assertLeakDetected(); + }); + } + + public void testAssertAtLeastOnceWillNotLogWhenResolvedOrFailed() { + assumeTrue("assertAtLeastOnce will be a no-op when assertions are disabled", Assertions.ENABLED); + ReachabilityChecker reachabilityChecker = new ReachabilityChecker(); + ActionListener listenerRef = reachabilityChecker.register(ActionListener.assertAtLeastOnce(ActionListener.running(() -> { + // Do nothing, but don't use ActionListener.noop() as it'll never be garbage collected + }))); + // Call onResponse and/or onFailure at least once + int times = randomIntBetween(1, 3); + for (int i = 0; i < times; i++) { + if (randomBoolean()) { + listenerRef.onResponse("succeeded"); + } else { + listenerRef.onFailure(new RuntimeException("Failed")); + } + } + // Nullify reference so it becomes unreachable + listenerRef = null; + reachabilityChecker.ensureUnreachable(); + } + + public void testAssertAtLeastOnceWillDelegateResponses() { + final var response = new Object(); + assertSame(response, safeAwait(SubscribableListener.newForked(l -> ActionListener.assertAtLeastOnce(l).onResponse(response)))); + } + + public void testAssertAtLeastOnceWillDelegateFailures() { + final var exception = new RuntimeException(); + assertSame( + exception, + safeAwaitFailure(SubscribableListener.newForked(l -> ActionListener.assertAtLeastOnce(l).onFailure(exception))) + ); + } + /** * Test that map passes the output of the function to its delegate listener and that exceptions in the function are propagated to the * onFailure handler. Also verify that exceptions from ActionListener.onResponse does not invoke onFailure, since it is the diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java index 6ade8fc184ed9..ed81f6750aa27 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/allocation/ClusterAllocationExplanationTests.java @@ -165,8 +165,7 @@ private static ClusterAllocationExplanation randomClusterAllocationExplanation(b DiscoveryNode node = assignedShard ? DiscoveryNodeUtils.builder("node-0").roles(emptySet()).build() : null; ShardAllocationDecision shardAllocationDecision; if (assignedShard) { - MoveDecision moveDecision = MoveDecision.cannotRebalance(Decision.YES, AllocationDecision.NO, 3, null) - .withRemainDecision(Decision.YES); + MoveDecision moveDecision = MoveDecision.rebalance(Decision.YES, Decision.YES, AllocationDecision.NO, null, 3, null); shardAllocationDecision = new ShardAllocationDecision(AllocateUnassignedDecision.NOT_TAKEN, moveDecision); } else { AllocateUnassignedDecision allocateDecision = AllocateUnassignedDecision.no(UnassignedInfo.AllocationStatus.DECIDERS_NO, null); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/hotthreads/NodesHotThreadsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestTests.java similarity index 77% rename from server/src/test/java/org/elasticsearch/action/admin/cluster/hotthreads/NodesHotThreadsRequestTests.java rename to server/src/test/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestTests.java index 883e905d0d514..4b02ddf5b4b94 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/hotthreads/NodesHotThreadsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/hotthreads/NodesHotThreadsRequestTests.java @@ -6,10 +6,10 @@ * Side Public License, v 1. */ -package org.elasticsearch.action.admin.cluster.hotthreads; +package org.elasticsearch.action.admin.cluster.node.hotthreads; import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.core.TimeValue; @@ -25,12 +25,17 @@ public class NodesHotThreadsRequestTests extends ESTestCase { public void testBWCSerialization() throws IOException { TimeValue sampleInterval = new TimeValue(50, TimeUnit.MINUTES); - NodesHotThreadsRequest request = new NodesHotThreadsRequest("123"); - request.threads(4); - request.ignoreIdleThreads(false); - request.type(HotThreads.ReportType.BLOCK); - request.interval(sampleInterval); - request.snapshots(3); + NodesHotThreadsRequest request = new NodesHotThreadsRequest( + new String[] { "123" }, + new HotThreads.RequestOptions( + 4, + HotThreads.ReportType.BLOCK, + HotThreads.RequestOptions.DEFAULT.sortOrder(), + sampleInterval, + 3, + false + ) + ); TransportVersion latest = TransportVersion.current(); TransportVersion previous = TransportVersionUtils.randomVersionBetween( @@ -41,10 +46,13 @@ public void testBWCSerialization() throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.setTransportVersion(latest); - request.writeTo(out); + request.requestOptions.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { in.setTransportVersion(previous); - NodesHotThreadsRequest deserialized = new NodesHotThreadsRequest(in); + NodesHotThreadsRequest deserialized = new NodesHotThreadsRequest( + Strings.EMPTY_ARRAY, + HotThreads.RequestOptions.readFrom(in) + ); assertEquals(request.threads(), deserialized.threads()); assertEquals(request.ignoreIdleThreads(), deserialized.ignoreIdleThreads()); assertEquals(request.type(), deserialized.type()); @@ -56,10 +64,13 @@ public void testBWCSerialization() throws IOException { try (BytesStreamOutput out = new BytesStreamOutput()) { out.setTransportVersion(previous); - request.writeTo(out); + request.requestOptions.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { in.setTransportVersion(previous); - NodesHotThreadsRequest deserialized = new NodesHotThreadsRequest(in); + NodesHotThreadsRequest deserialized = new NodesHotThreadsRequest( + Strings.EMPTY_ARRAY, + HotThreads.RequestOptions.readFrom(in) + ); assertEquals(request.threads(), deserialized.threads()); assertEquals(request.ignoreIdleThreads(), deserialized.ignoreIdleThreads()); assertEquals(request.type(), deserialized.type()); diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java index edee453cad0ca..82f22a2572c1d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodeStatsTests.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.shard.ShardCountStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.index.stats.IndexingPressureStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.translog.TranslogStats; @@ -643,6 +644,7 @@ private static CommonStats createShardLevelCommonStats() { indicesCommonStats.getShards().add(new ShardCountStats(++iota)); indicesCommonStats.getDenseVectorStats().add(new DenseVectorStats(++iota)); + indicesCommonStats.getSparseVectorStats().add(new SparseVectorStats(++iota)); return indicesCommonStats; } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java index ad5f1e5034dd6..000f99b270df2 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/stats/NodesStatsRequestTests.java @@ -130,9 +130,9 @@ public void testUnknownMetricsRejected() { */ private static NodesStatsRequest roundTripRequest(NodesStatsRequest request) throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { - request.writeTo(out); + request.getNodesStatsRequestParameters().writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - return new NodesStatsRequest(in); + return new NodesStatsRequest(new NodesStatsRequestParameters(in), request.nodesIds()); } } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java index 9883eec6896ce..e541fef65a0f9 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java @@ -107,12 +107,6 @@ public CancellableNodesRequest(String requestName, String... nodesIds) { this.requestName = requestName; } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(requestName); - } - @Override public String getDescription() { return "CancellableNodesRequest[" + requestName + "]"; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java index 6f345eb7dcdab..859ee68a7846d 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TestTaskPlugin.java @@ -230,15 +230,6 @@ public boolean getShouldFail() { return shouldFail; } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(requestName); - out.writeBoolean(shouldStoreResult); - out.writeBoolean(shouldBlock); - out.writeBoolean(shouldFail); - } - @Override public String getDescription() { return "NodesRequest[" + requestName + "]"; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index 969ed50685bc2..ca885b081452b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -114,12 +114,6 @@ public NodesRequest(String requestName, String... nodesIds) { this.requestName = requestName; } - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeString(requestName); - } - @Override public String getDescription() { return "CancellableNodesRequest[" + requestName + "]"; diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java index 1b3e0ae6fe7bf..092b3289ab85c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/repositories/reservedstate/ReservedRepositoryActionTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.admin.cluster.repositories.reservedstate; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.RepositoryMetadata; @@ -20,9 +21,7 @@ import org.elasticsearch.repositories.RepositoryMissingException; import org.elasticsearch.reservedstate.TransformState; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.MockUtils; import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; @@ -134,15 +133,14 @@ public Repository create(RepositoryMetadata metadata) { }; ThreadPool threadPool = mock(ThreadPool.class); - TransportService transportService = MockUtils.setupTransportServiceWithThreadpoolExecutor(threadPool); RepositoriesService repositoriesService = spy( new RepositoriesService( Settings.EMPTY, mock(ClusterService.class), - transportService, Map.of(), Map.of("fs", fsFactory), threadPool, + mock(NodeClient.class), null ) ); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index c2edf9729b8b8..28c153c734b7c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -268,17 +268,6 @@ public void testDataStreamValidation() throws IOException { exception.getMessage(), equalTo("aliases, mappings, and index settings may not be specified when rolling over a data stream") ); - - exception = expectThrows( - IllegalArgumentException.class, - () -> MetadataRolloverService.validate(metadata, randomDataStream.getName(), null, req, true) - ); - assertThat( - exception.getMessage(), - equalTo( - "unable to roll over failure store because [" + randomDataStream.getName() + "] does not have the failure store enabled" - ) - ); } public void testGenerateRolloverIndexName() { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 2dfc6ca24f4ac..cd2ac1939872c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -59,6 +59,7 @@ import org.elasticsearch.index.shard.IndexingStats; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.shard.ShardPath; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.warmer.WarmerStats; import org.elasticsearch.indices.EmptySystemIndices; @@ -674,6 +675,7 @@ public static IndicesStatsResponse randomIndicesStatsResponse(final IndexMetadat stats.flush = new FlushStats(); stats.warmer = new WarmerStats(); stats.denseVectorStats = new DenseVectorStats(); + stats.sparseVectorStats = new SparseVectorStats(); shardStats.add(new ShardStats(shardRouting, new ShardPath(false, path, path, shardId), stats, null, null, null, false, 0)); } } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/CommonStatsTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/CommonStatsTests.java index cad32f3b38cd7..0645ae4f36e14 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/stats/CommonStatsTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/stats/CommonStatsTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.shard.DenseVectorStats; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.test.AbstractWireSerializingTestCase; import org.elasticsearch.test.ESTestCase; @@ -50,6 +51,7 @@ protected CommonStats createTestInstance() { @Override protected CommonStats mutateInstance(CommonStats instance) throws IOException { CommonStats another = createTestInstance(); + long denseVectorCount = instance.getDenseVectorStats() == null ? randomNonNegativeLong() : randomValueOtherThan(instance.getDenseVectorStats().getValueCount(), ESTestCase::randomNonNegativeLong); @@ -58,6 +60,16 @@ protected CommonStats mutateInstance(CommonStats instance) throws IOException { } else { another.getDenseVectorStats().add(new DenseVectorStats(denseVectorCount)); } + + long sparseVectorCount = instance.getSparseVectorStats() == null + ? randomNonNegativeLong() + : randomValueOtherThan(instance.getSparseVectorStats().getValueCount(), ESTestCase::randomNonNegativeLong); + if (another.getSparseVectorStats() == null) { + another.sparseVectorStats = new SparseVectorStats(sparseVectorCount); + } else { + another.getSparseVectorStats().add(new SparseVectorStats(sparseVectorCount)); + } + another.add(instance); return another; } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index 20d826b11c1e7..4ca4e7158e454 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.action.bulk; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; @@ -25,7 +26,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; @@ -146,7 +146,8 @@ void executeBulk( } @Override - void createIndex(String index, boolean requireDataStream, TimeValue timeout, ActionListener listener) { + void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { + String index = createIndexRequest.index(); try { simulateAutoCreate.accept(index); // If we try to create an index just immediately assume it worked diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 52d50b3a23a0d..d7adf3aa8b4e2 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; @@ -36,7 +37,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexSettings; @@ -174,7 +174,7 @@ void executeBulk( } @Override - void createIndex(String index, boolean requireDataStream, TimeValue timeout, ActionListener listener) { + void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { indexCreated = true; listener.onResponse(null); } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index c27263f43eff1..1a34b1e856a5e 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.bulk.TransportBulkActionTookTests.Resolver; import org.elasticsearch.action.delete.DeleteRequest; @@ -34,7 +35,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexVersion; @@ -105,7 +105,7 @@ class TestTransportBulkAction extends TransportBulkAction { } @Override - void createIndex(String index, boolean requireDataStream, TimeValue timeout, ActionListener listener) { + void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { indexCreated = true; if (beforeIndexCreation != null) { beforeIndexCreation.run(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java index 47a6a03078b9a..590029f8537f7 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportSimulateBulkActionTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.ingest.SimulateIndexResponse; @@ -22,7 +23,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.indices.EmptySystemIndices; @@ -83,7 +83,7 @@ class TestTransportSimulateBulkAction extends TransportSimulateBulkAction { } @Override - void createIndex(String index, boolean requireDataStream, TimeValue timeout, ActionListener listener) { + void createIndex(CreateIndexRequest createIndexRequest, ActionListener listener) { indexCreated = true; if (beforeIndexCreation != null) { beforeIndexCreation.run(); @@ -192,7 +192,7 @@ public void onFailure(Exception e) { fail(e, "Unexpected error"); } }; - Map indicesToAutoCreate = Map.of(); // unused + Map indicesToAutoCreate = Map.of(); // unused Set dataStreamsToRollover = Set.of(); // unused Set failureStoresToRollover = Set.of(); // unused long startTime = 0; diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java new file mode 100644 index 0000000000000..191505e3afbbf --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/support/nodes/BaseNodesXContentResponseTests.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.action.support.nodes; + +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ChunkedToXContentHelper; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.ToXContent; + +import java.util.Iterator; +import java.util.List; + +public class BaseNodesXContentResponseTests extends ESTestCase { + public void testFragmentFlag() { + final var node = DiscoveryNodeUtils.create("test"); + + class TestNodeResponse extends BaseNodeResponse { + protected TestNodeResponse(DiscoveryNode node) { + super(node); + } + } + + final var fullResponse = new BaseNodesXContentResponse<>(ClusterName.DEFAULT, List.of(new TestNodeResponse(node)), List.of()) { + @Override + protected Iterator xContentChunks(ToXContent.Params outerParams) { + return ChunkedToXContentHelper.singleChunk((b, p) -> b.startObject("content").endObject()); + } + + @Override + protected List readNodesFrom(StreamInput in) { + return TransportAction.localOnly(); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) { + TransportAction.localOnly(); + } + }; + + assertFalse(fullResponse.isFragment()); + + assertEquals(""" + { + "_nodes" : { + "total" : 1, + "successful" : 1, + "failed" : 0 + }, + "cluster_name" : "elasticsearch", + "content" : { } + }""", Strings.toString(fullResponse, true, false)); + } +} diff --git a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java index d0535665d3685..3913849095787 100644 --- a/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/nodes/TransportNodesActionTests.java @@ -391,8 +391,8 @@ private static class DataNodesOnlyTransportNodesAction extends TestTransportNode } @Override - protected void resolveRequest(TestNodesRequest request, ClusterState clusterState) { - request.setConcreteNodes(clusterState.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new)); + protected DiscoveryNode[] resolveRequest(TestNodesRequest request, ClusterState clusterState) { + return clusterState.nodes().getDataNodes().values().toArray(DiscoveryNode[]::new); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java index 79203899b665d..4b131cf5f81ee 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/JoinValidationServiceTests.java @@ -234,7 +234,7 @@ public void onFailure(Exception e) { for (final var thread : threads) { thread.join(); } - assertTrue(validationPermits.tryAcquire(permitCount, 10, TimeUnit.SECONDS)); + safeAcquire(permitCount, validationPermits); assertBusy(() -> assertTrue(joinValidationService.isIdle())); } finally { Collections.reverse(releasables); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java index 7e338c52a0a17..922a1405bddff 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamLifecycleWithRetentionWarningsTests.java @@ -191,6 +191,48 @@ public void testValidateLifecycleIndexTemplateWithWarning() { ); } + /** + * Make sure we still take into account component templates during validation (and not just the index template). + */ + public void testValidateLifecycleComponentTemplateWithWarning() { + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + HeaderWarning.setThreadContext(threadContext); + TimeValue defaultRetention = randomTimeValue(2, 100, TimeUnit.DAYS); + MetadataIndexTemplateService.validateLifecycle( + Metadata.builder() + .componentTemplates( + Map.of( + "component-template", + new ComponentTemplate( + new Template( + null, + null, + null, + new DataStreamLifecycle( + new DataStreamLifecycle.Retention(randomTimeValue(2, 100, TimeUnit.DAYS)), + null, + null + ) + ), + null, + null + ) + ) + ) + .build(), + randomAlphaOfLength(10), + ComposableIndexTemplate.builder() + .template(new Template(null, null, null, DataStreamLifecycle.DEFAULT)) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .indexPatterns(List.of(randomAlphaOfLength(10))) + .componentTemplates(List.of("component-template")) + .build(), + new DataStreamGlobalRetention(defaultRetention, null) + ); + Map> responseHeaders = threadContext.getResponseHeaders(); + assertThat(responseHeaders.size(), is(0)); + } + public void testValidateLifecycleInComponentTemplate() throws Exception { IndicesService indicesService = mock(IndicesService.class); IndexService indexService = mock(IndexService.class); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 0277855db9c4c..07481a68c5176 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -353,24 +353,17 @@ public void testRemoveFailureStoreIndexThatDoesNotExist() { public void testRemoveFailureStoreWriteIndex() { DataStream original = createRandomDataStream(); + int indexToRemove = original.getFailureIndices().getIndices().size() - 1; - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> original.removeFailureStoreIndex( - original.getFailureIndices().getIndices().get(original.getFailureIndices().getIndices().size() - 1) - ) - ); - assertThat( - e.getMessage(), - equalTo( - String.format( - Locale.ROOT, - "cannot remove backing index [%s] of data stream [%s] because it is the write index of the failure store", - original.getFailureIndices().getIndices().get(original.getFailureIndices().getIndices().size() - 1).getName(), - original.getName() - ) - ) - ); + DataStream updated = original.removeFailureStoreIndex(original.getFailureIndices().getIndices().get(indexToRemove)); + assertThat(updated.getName(), equalTo(original.getName())); + assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1)); + assertThat(updated.getIndices().size(), equalTo(original.getIndices().size())); + assertThat(updated.getFailureIndices().getIndices().size(), equalTo(original.getFailureIndices().getIndices().size() - 1)); + assertThat(updated.getFailureIndices().isRolloverOnWrite(), equalTo(true)); + for (int k = 0; k < (original.getFailureIndices().getIndices().size() - 1); k++) { + assertThat(updated.getFailureIndices().getIndices().get(k), equalTo(original.getFailureIndices().getIndices().get(k))); + } } public void testAddBackingIndex() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index f172d0e21743d..c900c3257a405 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -38,9 +38,11 @@ import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.newInstance; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.mock; @@ -65,7 +67,8 @@ public void testCreateDataStream() throws Exception { cs, true, req, - ActionListener.noop() + ActionListener.noop(), + false ); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); @@ -105,7 +108,8 @@ public void testCreateDataStreamWithAliasFromTemplate() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); @@ -182,7 +186,8 @@ public void testCreateDataStreamWithAliasFromComponentTemplate() throws Exceptio cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); @@ -219,7 +224,7 @@ private static AliasMetadata randomAlias(String prefix) { return builder.build(); } - public void testCreateDataStreamWithFailureStore() throws Exception { + public void testCreateDataStreamWithFailureStoreInitialized() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStreamName + "*")) @@ -235,7 +240,8 @@ public void testCreateDataStreamWithFailureStore() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + true ); var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); @@ -252,6 +258,39 @@ public void testCreateDataStreamWithFailureStore() throws Exception { assertThat(newState.metadata().index(failureStoreIndexName).isSystem(), is(false)); } + public void testCreateDataStreamWithFailureStoreUninitialized() throws Exception { + final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); + final String dataStreamName = "my-data-stream"; + ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of(dataStreamName + "*")) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, true)) + .build(); + ClusterState cs = ClusterState.builder(new ClusterName("_name")) + .metadata(Metadata.builder().put("template", template).build()) + .build(); + CreateDataStreamClusterStateUpdateRequest req = new CreateDataStreamClusterStateUpdateRequest(dataStreamName); + ClusterState newState = MetadataCreateDataStreamService.createDataStream( + metadataCreateIndexService, + Settings.EMPTY, + cs, + randomBoolean(), + req, + ActionListener.noop(), + false + ); + var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); + var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); + assertThat(newState.metadata().dataStreams().size(), equalTo(1)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false)); + assertThat(newState.metadata().dataStreams().get(dataStreamName).getFailureIndices().getIndices(), empty()); + assertThat(newState.metadata().index(backingIndexName), notNullValue()); + assertThat(newState.metadata().index(backingIndexName).getSettings().get("index.hidden"), equalTo("true")); + assertThat(newState.metadata().index(backingIndexName).isSystem(), is(false)); + assertThat(newState.metadata().index(failureStoreIndexName), nullValue()); + } + public void testCreateDataStreamWithFailureStoreWithRefreshRate() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); var timeValue = randomTimeValue(); @@ -272,7 +311,8 @@ public void testCreateDataStreamWithFailureStoreWithRefreshRate() throws Excepti cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + true ); var backingIndexName = DataStream.getDefaultBackingIndexName(dataStreamName, 1, req.getStartTime()); var failureStoreIndexName = DataStream.getDefaultFailureStoreName(dataStreamName, 1, req.getStartTime()); @@ -303,7 +343,8 @@ public void testCreateSystemDataStream() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ); assertThat(newState.metadata().dataStreams().size(), equalTo(1)); assertThat(newState.metadata().dataStreams().get(dataStreamName).getName(), equalTo(dataStreamName)); @@ -336,7 +377,8 @@ public void testCreateDuplicateDataStream() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ) ); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] already exists")); @@ -355,7 +397,8 @@ public void testCreateDataStreamWithInvalidName() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ) ); assertThat(e.getMessage(), containsString("must not contain the following characters")); @@ -374,7 +417,8 @@ public void testCreateDataStreamWithUppercaseCharacters() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ) ); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must be lowercase")); @@ -393,7 +437,8 @@ public void testCreateDataStreamStartingWithPeriod() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ) ); assertThat(e.getMessage(), containsString("data_stream [" + dataStreamName + "] must not start with '.ds-'")); @@ -412,7 +457,8 @@ public void testCreateDataStreamNoTemplate() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ) ); assertThat(e.getMessage(), equalTo("no matching index template found for data stream [my-data-stream]")); @@ -434,7 +480,8 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ) ); assertThat( @@ -459,7 +506,8 @@ public static ClusterState createDataStream(final String dataStreamName) throws cs, randomBoolean(), req, - ActionListener.noop() + ActionListener.noop(), + false ); } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetTests.java new file mode 100644 index 0000000000000..347b5365c6f72 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/AllocationFailuresResetTests.java @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.cluster.routing.allocation; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.EmptyClusterInfoService; +import org.elasticsearch.cluster.TestShardRoutingRoleStrategies; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.IndexShardRoutingTable; +import org.elasticsearch.cluster.routing.RecoverySource; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.routing.UnassignedInfo; +import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; +import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.EmptySnapshotsInfoService; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.gateway.TestGatewayAllocator; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; + +import java.util.List; +import java.util.Set; + +public class AllocationFailuresResetTests extends ESTestCase { + + private ThreadPool threadPool; + private ClusterService clusterService; + + private static ClusterState addNode(ClusterState state, String name) { + var nodes = DiscoveryNodes.builder(state.nodes()).add(DiscoveryNodeUtils.create(name)); + return ClusterState.builder(state).nodes(nodes).build(); + } + + private static ClusterState removeNode(ClusterState state, String name) { + var nodes = DiscoveryNodes.builder(); + state.nodes().stream().filter((node) -> node.getId() != name).forEach(nodes::add); + return ClusterState.builder(state).nodes(nodes).build(); + } + + private static ClusterState addShardWithFailures(ClusterState state) { + var index = "index-1"; + var shard = 0; + + var indexMeta = new IndexMetadata.Builder(index).settings( + Settings.builder().put(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build()) + ).numberOfShards(1).numberOfReplicas(0).build(); + + var meta = Metadata.builder(state.metadata()).put(indexMeta, false).build(); + + var shardId = new ShardId(indexMeta.getIndex(), shard); + var nonZeroFailures = 5; + var unassignedInfo = new UnassignedInfo( + UnassignedInfo.Reason.ALLOCATION_FAILED, + null, + null, + nonZeroFailures, + 0, + 0, + false, + UnassignedInfo.AllocationStatus.NO_ATTEMPT, + Set.of(), + null + ); + + var shardRouting = ShardRouting.newUnassigned( + shardId, + true, + new RecoverySource.EmptyStoreRecoverySource(), + unassignedInfo, + ShardRouting.Role.DEFAULT + ); + + var routingTable = new RoutingTable.Builder().add( + new IndexRoutingTable.Builder(TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY, indexMeta.getIndex()).initializeAsNew( + meta.index(index) + ).addIndexShard(IndexShardRoutingTable.builder(shardId).addShard(shardRouting)).build() + ).build(); + + return ClusterState.builder(state).metadata(meta).routingTable(routingTable).build(); + } + + @Override + public void setUp() throws Exception { + super.setUp(); + threadPool = new TestThreadPool("reset-alloc-failures"); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + var allocationService = new AllocationService( + new AllocationDeciders(List.of(new MaxRetryAllocationDecider())), + new TestGatewayAllocator(), + new BalancedShardsAllocator(Settings.EMPTY), + EmptyClusterInfoService.INSTANCE, + EmptySnapshotsInfoService.INSTANCE, + TestShardRoutingRoleStrategies.DEFAULT_ROLE_ONLY + ); + allocationService.addAllocFailuresResetListenerTo(clusterService); + } + + @Override + public void tearDown() throws Exception { + super.tearDown(); + clusterService.stop(); + threadPool.shutdownNow(); + } + + /** + * Create state with two nodes and allocation failures, and does not reset counter after node removal + */ + public void testRemoveNodeDoesNotResetCounter() throws Exception { + var initState = clusterService.state(); + var stateWithNewNode = addNode(initState, "node-2"); + clusterService.getClusterApplierService().onNewClusterState("add node", () -> stateWithNewNode, ActionListener.noop()); + + var stateWithFailures = addShardWithFailures(stateWithNewNode); + clusterService.getClusterApplierService().onNewClusterState("add failures", () -> stateWithFailures, ActionListener.noop()); + + assertBusy(() -> { + var resultState = clusterService.state(); + assertEquals(2, resultState.nodes().size()); + assertEquals(1, resultState.getRoutingTable().allShards().count()); + assertTrue(resultState.getRoutingNodes().hasAllocationFailures()); + }); + + var stateWithRemovedNode = removeNode(stateWithFailures, "node-2"); + clusterService.getClusterApplierService().onNewClusterState("remove node", () -> stateWithRemovedNode, ActionListener.noop()); + assertBusy(() -> { + var resultState = clusterService.state(); + assertEquals(1, resultState.nodes().size()); + assertEquals(1, resultState.getRoutingTable().allShards().count()); + assertTrue(resultState.getRoutingNodes().hasAllocationFailures()); + }); + } + + /** + * Create state with one node and allocation failures, and reset counter after node addition + */ + public void testAddNodeResetsCounter() throws Exception { + var initState = clusterService.state(); + var stateWithFailures = addShardWithFailures(initState); + clusterService.getClusterApplierService().onNewClusterState("add failures", () -> stateWithFailures, ActionListener.noop()); + + var stateWithNewNode = addNode(stateWithFailures, "node-2"); + clusterService.getClusterApplierService().onNewClusterState("add node", () -> stateWithNewNode, ActionListener.noop()); + + assertBusy(() -> { + var resultState = clusterService.state(); + assertEquals(2, resultState.nodes().size()); + assertEquals(1, resultState.getRoutingTable().allShards().count()); + assertFalse(resultState.getRoutingNodes().hasAllocationFailures()); + }); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java index 7378ca56d2a4d..e6a2bb01db140 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/BalancedSingleShardTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; -import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.Balancer; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDeciders; import org.elasticsearch.cluster.routing.allocation.decider.Decision; @@ -41,7 +40,7 @@ import static org.hamcrest.Matchers.lessThan; /** - * Tests for balancing a single shard, see {@link Balancer#decideRebalance(ShardRouting)}. + * Tests for balancing a single shard. */ public class BalancedSingleShardTests extends ESAllocationTestCase { @@ -106,7 +105,7 @@ public Decision canRebalance(ShardRouting shardRouting, RoutingAllocation alloca assertNull(rebalanceDecision.getTargetNode()); assertEquals(1, rebalanceDecision.getClusterRebalanceDecision().getDecisions().size()); for (Decision subDecision : rebalanceDecision.getClusterRebalanceDecision().getDecisions()) { - assertEquals("foobar", ((Decision.Single) subDecision).getExplanation()); + assertEquals("foobar", subDecision.getExplanation()); } assertAssignedNodeRemainsSame(allocator, routingAllocation, shard); diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java index 6c33a8c6b89dc..dc4a420f4bd46 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/allocation/MoveDecisionTests.java @@ -28,27 +28,27 @@ public class MoveDecisionTests extends ESTestCase { public void testCachedDecisions() { // cached stay decision - MoveDecision stay1 = MoveDecision.stay(Decision.YES); - MoveDecision stay2 = MoveDecision.stay(Decision.YES); + MoveDecision stay1 = MoveDecision.remain(Decision.YES); + MoveDecision stay2 = MoveDecision.remain(Decision.YES); assertSame(stay1, stay2); // not in explain mode, so should use cached decision - stay1 = MoveDecision.stay(new Decision.Single(Type.YES, null, null, (Object[]) null)); - stay2 = MoveDecision.stay(new Decision.Single(Type.YES, null, null, (Object[]) null)); + stay1 = MoveDecision.remain(new Decision.Single(Type.YES, null, null, (Object[]) null)); + stay2 = MoveDecision.remain(new Decision.Single(Type.YES, null, null, (Object[]) null)); assertNotSame(stay1, stay2); // cached cannot move decision - stay1 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, null); - stay2 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, null); + stay1 = MoveDecision.move(Decision.NO, AllocationDecision.NO, null, null); + stay2 = MoveDecision.move(Decision.NO, AllocationDecision.NO, null, null); assertSame(stay1, stay2); // final decision is YES, so shouldn't use cached decision DiscoveryNode node1 = DiscoveryNodeUtils.builder("node1").roles(emptySet()).build(); - stay1 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.YES, node1, null); - stay2 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.YES, node1, null); + stay1 = MoveDecision.move(Decision.NO, AllocationDecision.YES, node1, null); + stay2 = MoveDecision.move(Decision.NO, AllocationDecision.YES, node1, null); assertNotSame(stay1, stay2); assertEquals(stay1.getTargetNode(), stay2.getTargetNode()); // final decision is NO, but in explain mode, so shouldn't use cached decision - stay1 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, new ArrayList<>()); - stay2 = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, new ArrayList<>()); + stay1 = MoveDecision.move(Decision.NO, AllocationDecision.NO, null, new ArrayList<>()); + stay2 = MoveDecision.move(Decision.NO, AllocationDecision.NO, null, new ArrayList<>()); assertNotSame(stay1, stay2); assertSame(stay1.getAllocationDecision(), stay2.getAllocationDecision()); assertNotNull(stay1.getExplanation()); @@ -56,14 +56,14 @@ public void testCachedDecisions() { } public void testStayDecision() { - MoveDecision stay = MoveDecision.stay(Decision.YES); + MoveDecision stay = MoveDecision.remain(Decision.YES); assertTrue(stay.canRemain()); assertFalse(stay.forceMove()); assertTrue(stay.isDecisionTaken()); assertNull(stay.getNodeDecisions()); assertEquals(AllocationDecision.NO_ATTEMPT, stay.getAllocationDecision()); - stay = MoveDecision.stay(Decision.YES); + stay = MoveDecision.remain(Decision.YES); assertTrue(stay.canRemain()); assertFalse(stay.forceMove()); assertTrue(stay.isDecisionTaken()); @@ -78,7 +78,7 @@ public void testDecisionWithNodeExplanations() { List nodeDecisions = new ArrayList<>(); nodeDecisions.add(new NodeAllocationResult(node1, nodeDecision, 2)); nodeDecisions.add(new NodeAllocationResult(node2, nodeDecision, 1)); - MoveDecision decision = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.NO, null, nodeDecisions); + MoveDecision decision = MoveDecision.move(Decision.NO, AllocationDecision.NO, null, nodeDecisions); assertNotNull(decision.getAllocationDecision()); assertNotNull(decision.getExplanation()); assertNotNull(decision.getNodeDecisions()); @@ -86,7 +86,7 @@ public void testDecisionWithNodeExplanations() { // both nodes have the same decision type but node2 has a higher weight ranking, so node2 comes first assertEquals("node2", decision.getNodeDecisions().iterator().next().getNode().getId()); - decision = MoveDecision.cannotRemain(Decision.NO, AllocationDecision.YES, node2, null); + decision = MoveDecision.move(Decision.NO, AllocationDecision.YES, node2, null); assertEquals("node2", decision.getTargetNode().getId()); } @@ -104,7 +104,7 @@ public void testSerialization() throws IOException { 1 ) ); - MoveDecision moveDecision = MoveDecision.cannotRemain( + MoveDecision moveDecision = MoveDecision.move( Decision.NO, AllocationDecision.fromDecisionType(finalDecision), assignedNode, diff --git a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java index bebfce3d14899..26faa295cf727 100644 --- a/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/service/MasterServiceTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.component.Lifecycle; @@ -77,6 +78,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; import static java.util.Collections.emptySet; @@ -93,6 +95,7 @@ import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.startsWith; public class MasterServiceTests extends ESTestCase { @@ -498,7 +501,7 @@ public void onFailure(Exception e) {} @Override public ClusterState execute(ClusterState currentState) { relativeTimeInMillis += TimeValue.timeValueSeconds(3).millis(); - return ClusterState.builder(currentState).incrementVersion().build(); + return ClusterState.builder(currentState).build(); } @Override @@ -1243,7 +1246,7 @@ public void onFailure(Exception e) { public ClusterState execute(ClusterState currentState) { relativeTimeInMillis += MasterService.MASTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(Settings.EMPTY).millis() + randomLongBetween(1, 1000000); - return ClusterState.builder(currentState).incrementVersion().build(); + return ClusterState.builder(currentState).build(); } @Override @@ -1277,7 +1280,7 @@ public void onFailure(Exception e) { masterService.submitUnbatchedStateUpdateTask("test5", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - return ClusterState.builder(currentState).incrementVersion().build(); + return ClusterState.builder(currentState).build(); } @Override @@ -1293,7 +1296,7 @@ public void onFailure(Exception e) { masterService.submitUnbatchedStateUpdateTask("test6", new ClusterStateUpdateTask() { @Override public ClusterState execute(ClusterState currentState) { - return ClusterState.builder(currentState).incrementVersion().build(); + return ClusterState.builder(currentState).build(); } @Override @@ -1927,9 +1930,9 @@ public boolean innerMatch(LogEvent event) { ); barrier.await(10, TimeUnit.SECONDS); - assertTrue(smallBatchExecutor.semaphore.tryAcquire(4, 10, TimeUnit.SECONDS)); - assertTrue(manySourceExecutor.semaphore.tryAcquire(2048, 10, TimeUnit.SECONDS)); - assertTrue(manyTasksPerSourceExecutor.semaphore.tryAcquire(2048, 10, TimeUnit.SECONDS)); + safeAcquire(4, smallBatchExecutor.semaphore); + safeAcquire(2048, manySourceExecutor.semaphore); + safeAcquire(2048, manyTasksPerSourceExecutor.semaphore); mockLog.assertAllExpectationsMatched(); } } @@ -2592,6 +2595,69 @@ public void onFailure(Exception e) { } } + public void testVersionNumberProtection() { + runVersionNumberProtectionTest( + currentState -> ClusterState.builder(currentState) + .version(randomFrom(currentState.version() - 1, currentState.version() + 1)) + .build() + ); + + runVersionNumberProtectionTest( + currentState -> currentState.copyAndUpdateMetadata( + b -> b.version(randomFrom(currentState.metadata().version() - 1, currentState.metadata().version() + 1)) + ) + ); + + runVersionNumberProtectionTest( + currentState -> ClusterState.builder(currentState) + .routingTable( + RoutingTable.builder(currentState.routingTable()) + .version(randomFrom(currentState.routingTable().version() - 1, currentState.routingTable().version() + 1)) + .build() + ) + .build() + ); + } + + private void runVersionNumberProtectionTest(UnaryOperator updateOperator) { + final var deterministicTaskQueue = new DeterministicTaskQueue(); + final var threadPool = deterministicTaskQueue.getThreadPool(); + final var threadContext = threadPool.getThreadContext(); + final var failureCaught = new AtomicBoolean(); + + try ( + var masterService = createMasterService(true, null, threadPool, deterministicTaskQueue.getPrioritizedEsThreadPoolExecutor()); + var ignored = threadContext.stashContext() + ) { + final var taskId = randomIdentifier(); + + masterService.submitUnbatchedStateUpdateTask(taskId, new ClusterStateUpdateTask() { + @Override + public ClusterState execute(ClusterState currentState) { + return updateOperator.apply(currentState); + } + + @Override + public void onFailure(Exception e) { + assertThat( + asInstanceOf(IllegalStateException.class, e).getMessage(), + allOf(startsWith("cluster state update executor did not preserve version numbers"), containsString(taskId)) + ); + assertTrue(failureCaught.compareAndSet(false, true)); + } + }); + + // suppress assertion errors to check production behaviour + threadContext.putTransient(MasterService.TEST_ONLY_EXECUTOR_MAY_CHANGE_VERSION_NUMBER_TRANSIENT_NAME, new Object()); + threadContext.markAsSystemContext(); + deterministicTaskQueue.runAllRunnableTasks(); + assertFalse(deterministicTaskQueue.hasRunnableTasks()); + assertFalse(deterministicTaskQueue.hasDeferredTasks()); + + assertTrue(failureCaught.get()); + } + } + /** * Returns the cluster state that the master service uses (and that is provided by the discovery layer) */ diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessorTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessorTests.java index 5146dcea84253..65bcb473f7d22 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/AsyncIOProcessorTests.java @@ -78,7 +78,7 @@ public void run() { for (int i = 0; i < thread.length; i++) { thread[i].join(); } - assertTrue(semaphore.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS)); + safeAcquire(10, semaphore); assertEquals(count * thread.length, received.get()); } @@ -131,7 +131,7 @@ public void run() { for (int i = 0; i < thread.length; i++) { thread[i].join(); } - assertTrue(semaphore.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS)); + safeAcquire(Integer.MAX_VALUE, semaphore); assertEquals(count * thread.length, received.get()); assertEquals(actualFailed.get(), failed.get()); } diff --git a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java index e267b6c3b737d..e6c7733790b5f 100644 --- a/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/concurrent/EsExecutorsTests.java @@ -35,6 +35,7 @@ import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.nullValue; @@ -655,6 +656,55 @@ public void testParseExecutorName() throws InterruptedException { } } + public void testScalingWithTaskTimeTracking() { + final int min = between(1, 3); + final int max = between(min + 1, 6); + + { + ThreadPoolExecutor pool = EsExecutors.newScaling( + getClass().getName() + "/" + getTestName(), + min, + max, + between(1, 100), + randomTimeUnit(), + randomBoolean(), + EsExecutors.daemonThreadFactory("test"), + threadContext, + new EsExecutors.TaskTrackingConfig(randomBoolean(), randomDoubleBetween(0.01, 0.1, true)) + ); + assertThat(pool, instanceOf(TaskExecutionTimeTrackingEsThreadPoolExecutor.class)); + } + + { + ThreadPoolExecutor pool = EsExecutors.newScaling( + getClass().getName() + "/" + getTestName(), + min, + max, + between(1, 100), + randomTimeUnit(), + randomBoolean(), + EsExecutors.daemonThreadFactory("test"), + threadContext + ); + assertThat(pool, instanceOf(EsThreadPoolExecutor.class)); + } + + { + ThreadPoolExecutor pool = EsExecutors.newScaling( + getClass().getName() + "/" + getTestName(), + min, + max, + between(1, 100), + randomTimeUnit(), + randomBoolean(), + EsExecutors.daemonThreadFactory("test"), + threadContext, + DO_NOT_TRACK + ); + assertThat(pool, instanceOf(EsThreadPoolExecutor.class)); + } + } + private static void runRejectOnShutdownTest(ExecutorService executor) { for (int i = between(0, 10); i > 0; i--) { final var delayMillis = between(0, 100); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java index 2136b154480ff..0524412cff70b 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayServiceTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.node.DiscoveryNodes; import org.elasticsearch.cluster.routing.RerouteService; +import org.elasticsearch.cluster.routing.RoutingTable; import org.elasticsearch.cluster.routing.ShardRoutingRoleStrategy; import org.elasticsearch.cluster.service.ClusterApplierService; import org.elasticsearch.cluster.service.ClusterService; @@ -494,12 +495,22 @@ public void onFailure(Exception e) { private MasterServiceTaskQueue createSetClusterStateTaskQueue(ClusterService clusterService) { return clusterService.createTaskQueue("set-cluster-state", Priority.NORMAL, batchExecutionContext -> { - ClusterState targetState = batchExecutionContext.initialState(); + final var initialState = batchExecutionContext.initialState(); + var targetState = initialState; for (var taskContext : batchExecutionContext.taskContexts()) { targetState = taskContext.getTask().clusterState(); taskContext.success(() -> {}); } - return targetState; + // fix up the version numbers + final var finalStateBuilder = ClusterState.builder(targetState) + .version(initialState.version()) + .metadata(Metadata.builder(targetState.metadata()).version(initialState.metadata().version())); + if (initialState.clusterRecovered() || targetState.clusterRecovered() == false) { + finalStateBuilder.routingTable( + RoutingTable.builder(targetState.routingTable()).version(initialState.routingTable().version()) + ); + } + return finalStateBuilder.build(); }); } } diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 00de132f9200e..a89ac5bc5b74e 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -3639,7 +3639,8 @@ public void testRecoverFromForeignTranslog() throws IOException { null, config.getRelativeTimeInNanosSupplier(), null, - true + true, + config.getMapperService() ); expectThrows(EngineCreationFailureException.class, () -> new InternalEngine(brokenConfig)); @@ -7320,7 +7321,8 @@ public void testNotWarmUpSearcherInEngineCtor() throws Exception { config.getLeafSorter(), config.getRelativeTimeInNanosSupplier(), config.getIndexCommitListener(), - config.isPromotableToPrimary() + config.isPromotableToPrimary(), + config.getMapperService() ); try (InternalEngine engine = createEngine(configWithWarmer)) { assertThat(warmedUpReaders, empty()); @@ -7737,6 +7739,80 @@ public void testFlushListener() throws Exception { } } + public void testFlushListenerWithConcurrentIndexing() throws IOException, InterruptedException { + engine.close(); + final var barrierReference = new AtomicReference(); + engine = new InternalTestEngine(engine.config()) { + @Override + protected void commitIndexWriter(IndexWriter writer, Translog translog) throws IOException { + final CyclicBarrier barrier = barrierReference.get(); + if (barrier != null) { + safeAwait(barrier); + safeAwait(barrier); + } + super.commitIndexWriter(writer, translog); + if (barrier != null) { + safeAwait(barrier); + safeAwait(barrier); + } + } + }; + recoverFromTranslog(engine, translogHandler, Long.MAX_VALUE); + final var barrier = new CyclicBarrier(2); + barrierReference.set(barrier); + + // (1) Indexing the 1st doc before flush and it should be visible after flush + final Engine.IndexResult result1 = engine.index(indexForDoc(createParsedDoc(randomIdentifier(), null))); + final PlainActionFuture future1 = new PlainActionFuture<>(); + engine.addFlushListener(result1.getTranslogLocation(), future1); + assertFalse(future1.isDone()); + final Thread flushThread = new Thread(() -> engine.flush()); + flushThread.start(); + + // (2) Wait till flush thread block before commitIndexWriter and indexing the 2nd doc + safeAwait(barrier); + final Engine.IndexResult result2 = engine.index(indexForDoc(createParsedDoc(randomIdentifier(), null))); + final PlainActionFuture future2 = new PlainActionFuture<>(); + engine.addFlushListener(result2.getTranslogLocation(), future2); + assertFalse(future2.isDone()); + + // Let flush completes the commit + safeAwait(barrier); + safeAwait(barrier); + + // Randomly indexing the 3rd doc after commit. + final PlainActionFuture future3; + final boolean indexingAfterCommit = randomBoolean(); + if (indexingAfterCommit) { + final Engine.IndexResult result3 = engine.index(indexForDoc(createParsedDoc(randomIdentifier(), null))); + future3 = new PlainActionFuture<>(); + engine.addFlushListener(result3.getTranslogLocation(), future3); + assertFalse(future3.isDone()); + } else { + future3 = null; + } + safeAwait(barrier); + flushThread.join(); + + // The translog location before flush (1st doc) is always visible + assertThat(safeGet(future1), equalTo(engine.getLastCommittedSegmentInfos().getGeneration())); + + if (indexingAfterCommit) { + // Indexing after the commit makes indexWriter.hasUncommittedChanges() return true which in turn makes + // it unsafe to advance flushListener's commitLocation after commit. That is, the flushListener + // will not learn the translog location of the 2nd doc. + assertFalse(future2.isDone()); + // It requires a 2nd flush to make all translog locations to be visible + barrierReference.set(null); // remove the flush barrier + engine.flush(); + assertThat(safeGet(future2), equalTo(engine.getLastCommittedSegmentInfos().getGeneration())); + assertThat(safeGet(future3), equalTo(engine.getLastCommittedSegmentInfos().getGeneration())); + } else { + // If no indexing after commit, translog location of the 2nd doc should be visible. + assertThat(safeGet(future2), equalTo(engine.getLastCommittedSegmentInfos().getGeneration())); + } + } + private static void assertCommitGenerations(Map commits, List expectedGenerations) { assertCommitGenerations(commits.values().stream().map(Engine.IndexCommitRef::getIndexCommit).toList(), expectedGenerations); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java index 2b96636d36a90..a0822141aea22 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java @@ -122,6 +122,8 @@ public void testMissingValues() throws IOException { geoPoints.getSupplier().setNextDocId(d); if (points[d].length > 0) { assertEquals(points[d][0], geoPoints.getValue()); + Exception e = expectThrows(IndexOutOfBoundsException.class, () -> geoPoints.get(geoPoints.size())); + assertEquals("A document doesn't have a value for a field at position [" + geoPoints.size() + "]!", e.getMessage()); } else { Exception e = expectThrows(IllegalStateException.class, () -> geoPoints.getValue()); assertEquals( diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java index a09639c0d90df..5fcb31cb3b64e 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesLongsTests.java @@ -35,6 +35,9 @@ public void testLongs() throws IOException { assertEquals(values[d][0], (long) longs.get(0)); assertEquals(values[d][0], longField.get(Long.MIN_VALUE)); assertEquals(values[d][0], longField.get(0, Long.MIN_VALUE)); + + Exception e = expectThrows(IndexOutOfBoundsException.class, () -> { long l = longs.get(longs.size()); }); + assertEquals("A document doesn't have a value for a field at position [" + longs.size() + "]!", e.getMessage()); } else { Exception e = expectThrows(IllegalStateException.class, longs::getValue); assertEquals( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index a5d705076561b..a389e803e66b6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.geo.GeoJson; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.core.Tuple; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; @@ -39,6 +38,8 @@ import java.util.Map; import java.util.Objects; import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; import static org.elasticsearch.geometry.utils.Geohash.stringEncode; import static org.elasticsearch.test.ListMatcher.matchesList; @@ -535,6 +536,8 @@ protected List exampleMalformedValues() { ), exampleMalformedValue("-,1.3").errorMatches("latitude must be a number"), exampleMalformedValue("1.3,-").errorMatches("longitude must be a number"), + exampleMalformedValue(b -> b.startObject().field("lat", 1.3).endObject()).errorMatches("Required [lon]"), + exampleMalformedValue(b -> b.startObject().field("lon", 1.3).endObject()).errorMatches("Required [lat]"), exampleMalformedValue(b -> b.startObject().field("lat", "NaN").field("lon", 1.2).endObject()).errorMatches("Required [lat]"), exampleMalformedValue(b -> b.startObject().field("lat", 1.2).field("lon", "NaN").endObject()).errorMatches("Required [lon]"), exampleMalformedValue("NaN,1.3").errorMatches("invalid latitude NaN; must be between -90.0 and 90.0"), @@ -603,7 +606,6 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed, boolean columnReader) { - assumeFalse("synthetic _source for geo_point doesn't support ignore_malformed", ignoreMalformed); return new SyntheticSourceSupport() { private final boolean ignoreZValue = usually(); private final GeoPoint nullValue = usually() ? null : randomGeoPoint(); @@ -611,41 +613,64 @@ protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed, @Override public SyntheticSourceExample example(int maxVals) { if (randomBoolean()) { - Tuple v = generateValue(); + Value v = generateValue(); + if (v.malformedOutput != null) { + return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping); + } + if (columnReader) { - return new SyntheticSourceExample(v.v1(), decode(encode(v.v2())), encode(v.v2()), this::mapping); + return new SyntheticSourceExample(v.input, decode(encode(v.output)), encode(v.output), this::mapping); } - return new SyntheticSourceExample(v.v1(), v.v2(), v.v2().toWKT(), this::mapping); + return new SyntheticSourceExample(v.input, v.output, v.output.toWKT(), this::mapping); + } - List> values = randomList(1, maxVals, this::generateValue); - // For the synthetic source tests, the results are sorted in order of encoded values, but for row-stride reader - // they are sorted in order of input, so we sort both input and expected here to support both types of tests - List> sorted = values.stream() - .sorted((a, b) -> Long.compare(encode(a.v2()), encode(b.v2()))) + List values = randomList(1, maxVals, this::generateValue); + List in = values.stream().map(Value::input).toList(); + + List outputFromDocValues = values.stream() + .filter(v -> v.malformedOutput == null) + .sorted((a, b) -> Long.compare(encode(a.output), encode(b.output))) + .map(Value::output) .toList(); - List in = sorted.stream().map(Tuple::v1).toList(); - List outList = sorted.stream().map(Tuple::v2).toList(); + + // Malformed values always come last in synthetic source + Stream malformedValues = values.stream().filter(v -> v.malformedOutput != null).map(Value::malformedOutput); + + List outList = Stream.concat(outputFromDocValues.stream(), malformedValues).toList(); Object out = outList.size() == 1 ? outList.get(0) : outList; if (columnReader) { // When reading doc-values, the block is a list of encoded longs - List outBlockList = outList.stream().map(this::encode).toList(); + List outBlockList = outputFromDocValues.stream().map(this::encode).toList(); Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; return new SyntheticSourceExample(in, out, outBlock, this::mapping); } else { - // When reading row-stride, the block is a list of WKT encoded BytesRefs - List outBlockList = outList.stream().map(GeoPoint::toWKT).toList(); + // When reading row-stride, the block is a list of WKT encoded BytesRefs. + // Values are ordered in order of input. + List outBlockList = values.stream().filter(v -> v.malformedOutput == null).map(v -> v.output.toWKT()).toList(); Object outBlock = outBlockList.size() == 1 ? outBlockList.get(0) : outBlockList; return new SyntheticSourceExample(in, out, outBlock, this::mapping); } } - private Tuple generateValue() { + private record Value(Object input, GeoPoint output, Object malformedOutput) {} + + private Value generateValue() { if (nullValue != null && randomBoolean()) { - return Tuple.tuple(null, nullValue); + return new Value(null, nullValue, null); + } + if (ignoreMalformed && randomBoolean()) { + // Different malformed values are tested in #exampleMalformedValues(). + // Here the goal is to test inputs that contain mixed valid and malformed values. + List> choices = List.of( + () -> "not a valid geohash " + randomAlphaOfLength(3), + () -> Map.of("one", 1, "two", List.of(2, 22, 222), "three", Map.of("three", 33)) + ); + Object v = randomFrom(choices).get(); + return new Value(v, null, v); } GeoPoint point = randomGeoPoint(); - return Tuple.tuple(randomGeoPointInput(point), point); + return new Value(randomGeoPointInput(point), point, null); } private GeoPoint randomGeoPoint() { @@ -694,6 +719,9 @@ private void mapping(XContentBuilder b) throws IOException { if (rarely()) { b.field("store", false); } + if (ignoreMalformed) { + b.field("ignore_malformed", true); + } } @Override @@ -706,10 +734,6 @@ public List invalidExample() throws IOException { new SyntheticSourceInvalidExample( equalTo("field [field] of type [geo_point] doesn't support synthetic source because it declares copy_to"), b -> b.field("type", "geo_point").field("copy_to", "foo") - ), - new SyntheticSourceInvalidExample( - equalTo("field [field] of type [geo_point] doesn't support synthetic source because it ignores malformed points"), - b -> b.field("type", "geo_point").field("ignore_malformed", true) ) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 71a0e001dc72a..e7f8a16c5cc10 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -87,8 +87,8 @@ public void testMultipleIgnoredFieldsRootObject() throws IOException { String stringValue = randomAlphaOfLength(20); String syntheticSource = getSyntheticSourceWithFieldLimit(b -> { b.field("boolean_value", booleanValue); - b.field("string_value", stringValue); b.field("int_value", intValue); + b.field("string_value", stringValue); }); assertEquals(String.format(Locale.ROOT, """ {"boolean_value":%s,"int_value":%s,"string_value":"%s"}""", booleanValue, intValue, stringValue), syntheticSource); @@ -626,6 +626,7 @@ public void testNestedObjectWithField() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { + b.field("store_array_source", true); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -647,6 +648,7 @@ public void testNestedObjectWithArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path").field("type", "nested"); { + b.field("store_array_source", true); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -679,6 +681,7 @@ public void testNestedSubobjectWithField() throws IOException { b.startObject("int_value").field("type", "integer").endObject(); b.startObject("to").field("type", "nested"); { + b.field("store_array_source", true); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -719,6 +722,7 @@ public void testNestedSubobjectWithArray() throws IOException { b.startObject("int_value").field("type", "integer").endObject(); b.startObject("to").field("type", "nested"); { + b.field("store_array_source", true); b.startObject("properties"); { b.startObject("foo").field("type", "keyword").endObject(); @@ -758,7 +762,7 @@ public void testNestedSubobjectWithArray() throws IOException { public void testNestedObjectIncludeInRoot() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { - b.startObject("path").field("type", "nested").field("include_in_root", true); + b.startObject("path").field("type", "nested").field("store_array_source", true).field("include_in_root", true); { b.startObject("properties"); { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 412077b659b98..c767429d4c0fb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -29,6 +29,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.function.Function; import static org.hamcrest.Matchers.containsString; @@ -1567,6 +1568,175 @@ public void testNestedMapperFilters() throws Exception { assertThat(mapper2.parentTypeFilter(), equalTo(mapper1.nestedTypeFilter())); } + public void testStoreArraySourceinSyntheticSourceMode() throws IOException { + DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { + b.startObject("o").field("type", "nested").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + })); + assertNotNull(mapper.mapping().getRoot().getMapper("o")); + } + + public void testStoreArraySourceThrowsInNonSyntheticSourceMode() { + var exception = expectThrows(MapperParsingException.class, () -> createDocumentMapper(mapping(b -> { + b.startObject("o").field("type", "nested").field(ObjectMapper.STORE_ARRAY_SOURCE_PARAM, true).endObject(); + }))); + assertEquals("Parameter [store_array_source] can only be set in synthetic source mode.", exception.getMessage()); + } + + public void testSyntheticNestedWithObject() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").field("type", "nested"); + { + b.startObject("properties"); + { + b.startObject("foo").field("type", "keyword").endObject(); + b.startObject("bar").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource( + documentMapper, + b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } + ); + assertEquals(""" + {"path":{"bar":"B","foo":"A"}}""", syntheticSource); + } + + public void testSyntheticNestedWithArray() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").field("type", "nested"); + { + b.startObject("properties"); + { + b.startObject("foo").field("type", "keyword").endObject(); + b.startObject("bar").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + { + b.startObject().field("foo", "A").field("bar", "B").endObject(); + b.startObject().field("foo", "C").field("bar", "D").endObject(); + } + b.endArray(); + }); + assertEquals(""" + {"path":[{"bar":"B","foo":"A"},{"bar":"D","foo":"C"}]}""", syntheticSource); + } + + public void testSyntheticNestedWithSubObjects() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("boolean_value").field("type", "boolean").endObject(); + b.startObject("path"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("int_value").field("type", "integer").endObject(); + b.startObject("to").field("type", "nested"); + { + b.startObject("properties"); + { + b.startObject("foo").field("type", "keyword").endObject(); + b.startObject("bar").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })).documentMapper(); + + boolean booleanValue = randomBoolean(); + int intValue = randomInt(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.field("boolean_value", booleanValue); + b.startObject("path"); + { + b.field("int_value", intValue); + b.startObject("to").field("foo", "A").field("bar", "B").endObject(); + } + b.endObject(); + }); + assertEquals(String.format(Locale.ROOT, """ + {"boolean_value":%s,"path":{"int_value":%s,"to":{"bar":"B","foo":"A"}}}""", booleanValue, intValue), syntheticSource); + } + + public void testSyntheticNestedWithSubArrays() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("boolean_value").field("type", "boolean").endObject(); + b.startObject("path"); + { + b.field("type", "object"); + b.startObject("properties"); + { + b.startObject("int_value").field("type", "integer").endObject(); + b.startObject("to").field("type", "nested"); + { + b.startObject("properties"); + { + b.startObject("foo").field("type", "keyword").endObject(); + b.startObject("bar").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })).documentMapper(); + + boolean booleanValue = randomBoolean(); + int intValue = randomInt(); + var syntheticSource = syntheticSource(documentMapper, b -> { + b.field("boolean_value", booleanValue); + b.startObject("path"); + { + b.field("int_value", intValue); + b.startArray("to"); + { + b.startObject().field("foo", "A").field("bar", "B").endObject(); + b.startObject().field("foo", "C").field("bar", "D").endObject(); + } + b.endArray(); + } + b.endObject(); + }); + assertEquals( + String.format(Locale.ROOT, """ + {"boolean_value":%s,"path":{"int_value":%s,"to":[{"bar":"B","foo":"A"},{"bar":"D","foo":"C"}]}}""", booleanValue, intValue), + syntheticSource + ); + } + + public void testSyntheticNestedWithIncludeInRoot() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").field("type", "nested").field("include_in_root", true); + { + b.startObject("properties"); + { + b.startObject("foo").field("type", "keyword").endObject(); + b.startObject("bar").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + })).documentMapper(); + var syntheticSource = syntheticSource( + documentMapper, + b -> { b.startObject("path").field("foo", "A").field("bar", "B").endObject(); } + ); + assertEquals(""" + {"path":{"bar":"B","foo":"A"}}""", syntheticSource); + } + private NestedObjectMapper createNestedObjectMapperWithAllParametersSet(CheckedConsumer propertiesBuilder) throws IOException { DocumentMapper mapper = createDocumentMapper(mapping(b -> { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java index 53d6d4f150776..6c3f2e19ad4b1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapperTests.java @@ -642,25 +642,27 @@ public void testInvalidParameters() { e.getMessage(), containsString("Failed to parse mapping: Mapping definition for [field] has unsupported parameters: [foo : {}]") ); - e = expectThrows( - MapperParsingException.class, - () -> createDocumentMapper( - fieldMapping( - b -> b.field("type", "dense_vector") - .field("dims", 3) - .field("element_type", "byte") - .field("similarity", "l2_norm") - .field("index", true) - .startObject("index_options") - .field("type", "int8_hnsw") - .endObject() + for (String quantizationKind : new String[] { "int4_hnsw", "int8_hnsw", "int8_flat", "int4_flat" }) { + e = expectThrows( + MapperParsingException.class, + () -> createDocumentMapper( + fieldMapping( + b -> b.field("type", "dense_vector") + .field("dims", 4) + .field("element_type", "byte") + .field("similarity", "l2_norm") + .field("index", true) + .startObject("index_options") + .field("type", quantizationKind) + .endObject() + ) ) - ) - ); - assertThat( - e.getMessage(), - containsString("Failed to parse mapping: [element_type] cannot be [byte] when using index type [int8_hnsw]") - ); + ); + assertThat( + e.getMessage(), + containsString("Failed to parse mapping: [element_type] cannot be [byte] when using index type [" + quantizationKind + "]") + ); + } } public void testInvalidParametersBeforeIndexedByDefault() { @@ -1226,6 +1228,46 @@ public void testKnnVectorsFormat() throws IOException { assertEquals(expectedString, knnVectorsFormat.toString()); } + public void testKnnQuantizedFlatVectorsFormat() throws IOException { + boolean setConfidenceInterval = randomBoolean(); + float confidenceInterval = (float) randomDoubleBetween(0.90f, 1.0f, true); + for (String quantizedFlatFormat : new String[] { "int8_flat", "int4_flat" }) { + MapperService mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "dense_vector"); + b.field("dims", 4); + b.field("index", true); + b.field("similarity", "dot_product"); + b.startObject("index_options"); + b.field("type", quantizedFlatFormat); + if (setConfidenceInterval) { + b.field("confidence_interval", confidenceInterval); + } + b.endObject(); + })); + CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE); + Codec codec = codecService.codec("default"); + KnnVectorsFormat knnVectorsFormat; + if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) { + assertThat(codec, instanceOf(PerFieldMapperCodec.class)); + knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); + } else { + assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); + knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); + } + String expectedString = "ES813Int8FlatVectorFormat(name=ES813Int8FlatVectorFormat, innerFormat=" + + "Lucene99ScalarQuantizedVectorsFormat(name=Lucene99ScalarQuantizedVectorsFormat," + + " confidenceInterval=" + + (setConfidenceInterval ? Float.toString(confidenceInterval) : (quantizedFlatFormat.equals("int4_flat") ? "0.0" : null)) + + ", bits=" + + (quantizedFlatFormat.equals("int4_flat") ? 4 : 7) + + ", compress=" + + quantizedFlatFormat.equals("int4_flat") + + ", flatVectorScorer=ScalarQuantizedVectorScorer(nonQuantizedDelegate=DefaultFlatVectorScorer())," + + " rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())))"; + assertEquals(expectedString, knnVectorsFormat.toString()); + } + } + public void testKnnQuantizedHNSWVectorsFormat() throws IOException { final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10); final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10); @@ -1262,11 +1304,68 @@ public void testKnnQuantizedHNSWVectorsFormat() throws IOException { + ", flatVectorFormat=ES814ScalarQuantizedVectorsFormat(" + "name=ES814ScalarQuantizedVectorsFormat, confidenceInterval=" + (setConfidenceInterval ? confidenceInterval : null) - + ", rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())" + + ", bits=7, compressed=false, rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())" + "))"; assertEquals(expectedString, knnVectorsFormat.toString()); } + public void testKnnHalfByteQuantizedHNSWVectorsFormat() throws IOException { + final int m = randomIntBetween(1, DEFAULT_MAX_CONN + 10); + final int efConstruction = randomIntBetween(1, DEFAULT_BEAM_WIDTH + 10); + boolean setConfidenceInterval = randomBoolean(); + float confidenceInterval = (float) randomDoubleBetween(0.90f, 1.0f, true); + MapperService mapperService = createMapperService(fieldMapping(b -> { + b.field("type", "dense_vector"); + b.field("dims", 4); + b.field("index", true); + b.field("similarity", "dot_product"); + b.startObject("index_options"); + b.field("type", "int4_hnsw"); + b.field("m", m); + b.field("ef_construction", efConstruction); + if (setConfidenceInterval) { + b.field("confidence_interval", confidenceInterval); + } + b.endObject(); + })); + CodecService codecService = new CodecService(mapperService, BigArrays.NON_RECYCLING_INSTANCE); + Codec codec = codecService.codec("default"); + KnnVectorsFormat knnVectorsFormat; + if (CodecService.ZSTD_STORED_FIELDS_FEATURE_FLAG.isEnabled()) { + assertThat(codec, instanceOf(PerFieldMapperCodec.class)); + knnVectorsFormat = ((PerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); + } else { + assertThat(codec, instanceOf(LegacyPerFieldMapperCodec.class)); + knnVectorsFormat = ((LegacyPerFieldMapperCodec) codec).getKnnVectorsFormatForField("field"); + } + String expectedString = "ES814HnswScalarQuantizedVectorsFormat(name=ES814HnswScalarQuantizedVectorsFormat, maxConn=" + + m + + ", beamWidth=" + + efConstruction + + ", flatVectorFormat=ES814ScalarQuantizedVectorsFormat(" + + "name=ES814ScalarQuantizedVectorsFormat, confidenceInterval=" + + (setConfidenceInterval ? confidenceInterval : 0.0f) + + ", bits=4, compressed=true, rawVectorFormat=Lucene99FlatVectorsFormat(vectorsScorer=DefaultFlatVectorScorer())" + + "))"; + assertEquals(expectedString, knnVectorsFormat.toString()); + } + + public void testInvalidVectorDimensions() { + for (String quantizedFlatFormat : new String[] { "int4_hnsw", "int4_flat" }) { + MapperParsingException e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { + b.field("type", "dense_vector"); + b.field("dims", 5); + b.field("element_type", "float"); + b.field("index", true); + b.field("similarity", "dot_product"); + b.startObject("index_options"); + b.field("type", quantizedFlatFormat); + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("only supports even dimensions")); + } + } + @Override protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 27adc72fb5ed8..fa4c8bb089855 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -42,14 +42,41 @@ public DenseVectorFieldTypeTests() { this.indexed = randomBoolean(); } + private DenseVectorFieldMapper.IndexOptions randomIndexOptionsNonQuantized() { + return randomFrom( + new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), + new DenseVectorFieldMapper.FlatIndexOptions() + ); + } + + private DenseVectorFieldMapper.IndexOptions randomIndexOptionsAll() { + return randomFrom( + new DenseVectorFieldMapper.HnswIndexOptions(randomIntBetween(1, 100), randomIntBetween(1, 10_000)), + new DenseVectorFieldMapper.Int8HnswIndexOptions( + randomIntBetween(1, 100), + randomIntBetween(1, 10_000), + randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)) + ), + new DenseVectorFieldMapper.Int4HnswIndexOptions( + randomIntBetween(1, 100), + randomIntBetween(1, 10_000), + randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true)) + ), + new DenseVectorFieldMapper.FlatIndexOptions(), + new DenseVectorFieldMapper.Int8FlatIndexOptions(randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true))), + new DenseVectorFieldMapper.Int4FlatIndexOptions(randomFrom((Float) null, 0f, (float) randomDoubleBetween(0.9, 1.0, true))) + ); + } + private DenseVectorFieldType createFloatFieldType() { return new DenseVectorFieldType( "f", IndexVersion.current(), DenseVectorFieldMapper.ElementType.FLOAT, - 5, + 6, indexed, VectorSimilarity.COSINE, + indexed ? randomIndexOptionsAll() : null, Collections.emptyMap() ); } @@ -62,6 +89,7 @@ private DenseVectorFieldType createByteFieldType() { 5, true, VectorSimilarity.COSINE, + randomIndexOptionsNonQuantized(), Collections.emptyMap() ); } @@ -113,7 +141,7 @@ public void testDocValueFormat() { public void testFetchSourceValue() throws IOException { DenseVectorFieldType fft = createFloatFieldType(); - List vector = List.of(0.0, 1.0, 2.0, 3.0, 4.0); + List vector = List.of(0.0, 1.0, 2.0, 3.0, 4.0, 6.0); assertEquals(vector, fetchSourceValue(fft, vector)); DenseVectorFieldType bft = createByteFieldType(); assertEquals(vector, fetchSourceValue(bft, vector)); @@ -123,6 +151,9 @@ public void testCreateNestedKnnQuery() { BitSetProducer producer = context -> null; int dims = randomIntBetween(2, 2048); + if (dims % 2 != 0) { + dims++; + } { DenseVectorFieldType field = new DenseVectorFieldType( "f", @@ -131,6 +162,7 @@ public void testCreateNestedKnnQuery() { dims, true, VectorSimilarity.COSINE, + randomIndexOptionsAll(), Collections.emptyMap() ); float[] queryVector = new float[dims]; @@ -148,6 +180,7 @@ public void testCreateNestedKnnQuery() { dims, true, VectorSimilarity.COSINE, + randomIndexOptionsNonQuantized(), Collections.emptyMap() ); byte[] queryVector = new byte[dims]; @@ -166,6 +199,9 @@ public void testCreateNestedKnnQuery() { public void testExactKnnQuery() { int dims = randomIntBetween(2, 2048); + if (dims % 2 != 0) { + dims++; + } { DenseVectorFieldType field = new DenseVectorFieldType( "f", @@ -174,6 +210,7 @@ public void testExactKnnQuery() { dims, true, VectorSimilarity.COSINE, + randomIndexOptionsAll(), Collections.emptyMap() ); float[] queryVector = new float[dims]; @@ -200,6 +237,7 @@ public void testExactKnnQuery() { dims, true, VectorSimilarity.COSINE, + randomIndexOptionsNonQuantized(), Collections.emptyMap() ); byte[] queryVector = new byte[dims]; @@ -225,14 +263,15 @@ public void testFloatCreateKnnQuery() { "f", IndexVersion.current(), DenseVectorFieldMapper.ElementType.FLOAT, - 3, + 4, false, VectorSimilarity.COSINE, + null, Collections.emptyMap() ); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> unindexedField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f }, 10, null, null, null) + () -> unindexedField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }, 10, null, null, null) ); assertThat(e.getMessage(), containsString("to perform knn search on field [f], its mapping must have [index] set to [true]")); @@ -240,14 +279,15 @@ public void testFloatCreateKnnQuery() { "f", IndexVersion.current(), DenseVectorFieldMapper.ElementType.FLOAT, - 3, + 4, true, VectorSimilarity.DOT_PRODUCT, + randomIndexOptionsAll(), Collections.emptyMap() ); e = expectThrows( IllegalArgumentException.class, - () -> dotProductField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f }, 10, null, null, null) + () -> dotProductField.createKnnQuery(new float[] { 0.3f, 0.1f, 1.0f, 0.0f }, 10, null, null, null) ); assertThat(e.getMessage(), containsString("The [dot_product] similarity can only be used with unit-length vectors.")); @@ -255,14 +295,15 @@ public void testFloatCreateKnnQuery() { "f", IndexVersion.current(), DenseVectorFieldMapper.ElementType.FLOAT, - 3, + 4, true, VectorSimilarity.COSINE, + randomIndexOptionsAll(), Collections.emptyMap() ); e = expectThrows( IllegalArgumentException.class, - () -> cosineField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f }, 10, null, null, null) + () -> cosineField.createKnnQuery(new float[] { 0.0f, 0.0f, 0.0f, 0.0f }, 10, null, null, null) ); assertThat(e.getMessage(), containsString("The [cosine] similarity does not support vectors with zero magnitude.")); } @@ -276,6 +317,7 @@ public void testCreateKnnQueryMaxDims() { 4096, true, VectorSimilarity.COSINE, + randomIndexOptionsAll(), Collections.emptyMap() ); float[] queryVector = new float[4096]; @@ -294,6 +336,7 @@ public void testCreateKnnQueryMaxDims() { 4096, true, VectorSimilarity.COSINE, + randomIndexOptionsNonQuantized(), Collections.emptyMap() ); byte[] queryVector = new byte[4096]; @@ -313,6 +356,7 @@ public void testByteCreateKnnQuery() { 3, false, VectorSimilarity.COSINE, + randomIndexOptionsNonQuantized(), Collections.emptyMap() ); IllegalArgumentException e = expectThrows( @@ -328,6 +372,7 @@ public void testByteCreateKnnQuery() { 3, true, VectorSimilarity.COSINE, + randomIndexOptionsNonQuantized(), Collections.emptyMap() ); e = expectThrows( diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 1813f2a2e1c03..d272aaab1b231 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -1546,7 +1546,7 @@ public void run() { thread[i].join(); } } - assertTrue(semaphore.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS)); + safeAcquire(Integer.MAX_VALUE, semaphore); closeShards(shard); } @@ -1604,7 +1604,7 @@ public void run() { for (int i = 0; i < thread.length; i++) { thread[i].join(); } - assertTrue(semaphore.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS)); + safeAcquire(Integer.MAX_VALUE, semaphore); assertEquals(shard.getLastKnownGlobalCheckpoint(), shard.getLastSyncedGlobalCheckpoint()); closeShards(shard); @@ -4820,7 +4820,8 @@ public void testCloseShardWhileEngineIsWarming() throws Exception { config.getLeafSorter(), config.getRelativeTimeInNanosSupplier(), config.getIndexCommitListener(), - config.isPromotableToPrimary() + config.isPromotableToPrimary(), + config.getMapperService() ); return new InternalEngine(configWithWarmer); }); diff --git a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java index 7f22c9f9ccc2a..2b333277e2d4a 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RefreshListenersTests.java @@ -156,7 +156,8 @@ public void onFailedEngine(String reason, @Nullable Exception e) { null, System::nanoTime, null, - true + true, + null ); engine = new InternalEngine(config); EngineTestCase.recoverFromTranslog(engine, (e, s) -> 0, Long.MAX_VALUE); diff --git a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java index 2baca5662161d..91e81dcabe9a4 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/SearchOperationListenerTests.java @@ -12,8 +12,8 @@ import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TestSearchContext; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequest.Empty; import java.lang.reflect.Proxy; import java.util.ArrayList; @@ -269,11 +269,11 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest assertEquals(0, validateSearchContext.get()); if (throwingListeners == 0) { - compositeListener.validateReaderContext(mock(ReaderContext.class), Empty.INSTANCE); + compositeListener.validateReaderContext(mock(ReaderContext.class), new EmptyRequest()); } else { RuntimeException expected = expectThrows( RuntimeException.class, - () -> compositeListener.validateReaderContext(mock(ReaderContext.class), Empty.INSTANCE) + () -> compositeListener.validateReaderContext(mock(ReaderContext.class), new EmptyRequest()) ); assertNull(expected.getMessage()); assertEquals(throwingListeners - 1, expected.getSuppressed().length); diff --git a/server/src/test/java/org/elasticsearch/index/shard/SparseVectorStatsTests.java b/server/src/test/java/org/elasticsearch/index/shard/SparseVectorStatsTests.java new file mode 100644 index 0000000000000..c20534a1e8469 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/SparseVectorStatsTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.shard; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; + +public class SparseVectorStatsTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return SparseVectorStats::new; + } + + @Override + protected SparseVectorStats createTestInstance() { + return new SparseVectorStats(randomNonNegativeLong()); + } + + @Override + protected SparseVectorStats mutateInstance(SparseVectorStats instance) { + return new SparseVectorStats(randomValueOtherThan(instance.getValueCount(), ESTestCase::randomNonNegativeLong)); + } +} diff --git a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java index 4c6d6f563b950..b271938110e3a 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndexingMemoryControllerTests.java @@ -236,7 +236,7 @@ public void testMinBufferSizes() { Settings.builder().put("indices.memory.index_buffer_size", "0.001%").put("indices.memory.min_index_buffer_size", "6mb").build() ); - assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(6, ByteSizeUnit.MB))); + assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(6, ByteSizeUnit.MB).getBytes())); } public void testNegativeMinIndexBufferSize() { @@ -288,7 +288,7 @@ public void testMaxBufferSizes() { Settings.builder().put("indices.memory.index_buffer_size", "90%").put("indices.memory.max_index_buffer_size", "6mb").build() ); - assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(6, ByteSizeUnit.MB))); + assertThat(controller.indexingBufferSize(), equalTo(new ByteSizeValue(6, ByteSizeUnit.MB).getBytes())); } public void testThrottling() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index 0ebcbff6bf863..0ddd3274bff29 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -528,16 +528,16 @@ private IndicesClusterStateService createIndicesClusterStateService( Collections.emptySet() ); final ClusterService clusterService = mock(ClusterService.class); + final NodeClient client = mock(NodeClient.class); final RepositoriesService repositoriesService = new RepositoriesService( settings, clusterService, - transportService, Collections.emptyMap(), Collections.emptyMap(), threadPool, + client, List.of() ); - final NodeClient client = mock(NodeClient.class); final PeerRecoveryTargetService recoveryTargetService = new PeerRecoveryTargetService( client, threadPool, diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java index c3cdfe3b8981f..4629577068c9d 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesModuleTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.repositories; +import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.MockBigArrays; @@ -33,6 +34,7 @@ public class RepositoriesModuleTests extends ESTestCase { private Environment environment; private NamedXContentRegistry contentRegistry; + private ThreadPool threadPool; private List repoPlugins = new ArrayList<>(); private RepositoryPlugin plugin1; private RepositoryPlugin plugin2; @@ -40,13 +42,14 @@ public class RepositoriesModuleTests extends ESTestCase { private TransportService transportService; private ClusterService clusterService; private RecoverySettings recoverySettings; + private NodeClient nodeClient; @Override public void setUp() throws Exception { super.setUp(); environment = mock(Environment.class); contentRegistry = mock(NamedXContentRegistry.class); - ThreadPool threadPool = mock(ThreadPool.class); + threadPool = mock(ThreadPool.class); transportService = mock(TransportService.class); when(transportService.getThreadPool()).thenReturn(threadPool); clusterService = mock(ClusterService.class); @@ -57,6 +60,7 @@ public void setUp() throws Exception { repoPlugins.add(plugin1); repoPlugins.add(plugin2); when(environment.settings()).thenReturn(Settings.EMPTY); + nodeClient = mock(NodeClient.class); } public void testCanRegisterTwoRepositoriesWithDifferentTypes() { @@ -73,7 +77,8 @@ public void testCanRegisterTwoRepositoriesWithDifferentTypes() { new RepositoriesModule( environment, repoPlugins, - transportService, + nodeClient, + threadPool, mock(ClusterService.class), MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, @@ -95,7 +100,8 @@ public void testCannotRegisterTwoRepositoriesWithSameTypes() { () -> new RepositoriesModule( environment, repoPlugins, - transportService, + nodeClient, + threadPool, clusterService, MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, @@ -118,7 +124,8 @@ public void testCannotRegisterTwoInternalRepositoriesWithSameTypes() { () -> new RepositoriesModule( environment, repoPlugins, - mock(TransportService.class), + nodeClient, + threadPool, clusterService, MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, @@ -142,7 +149,8 @@ public void testCannotRegisterNormalAndInternalRepositoriesWithSameTypes() { () -> new RepositoriesModule( environment, repoPlugins, - mock(TransportService.class), + nodeClient, + threadPool, clusterService, MockBigArrays.NON_RECYCLING_INSTANCE, contentRegistry, diff --git a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java index e156fae19f2ef..7be1dcdcf7b77 100644 --- a/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/RepositoriesServiceTests.java @@ -10,8 +10,10 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.cluster.ClusterChangedEvent; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -20,6 +22,7 @@ import org.elasticsearch.cluster.metadata.RepositoriesMetadata; import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeRole; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.UUIDs; @@ -54,6 +57,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; import java.util.function.BooleanSupplier; @@ -82,9 +86,22 @@ public void setUp() throws Exception { null, Collections.emptySet() ); - clusterService = ClusterServiceUtils.createClusterService(threadPool); + DiscoveryNode localNode = DiscoveryNodeUtils.builder("local").name("local").roles(Set.of(DiscoveryNodeRole.MASTER_ROLE)).build(); + NodeClient client = new NodeClient(Settings.EMPTY, threadPool); + var actionFilters = new ActionFilters(Set.of()); + client.initialize( + Map.of( + VerifyNodeRepositoryCoordinationAction.TYPE, + new VerifyNodeRepositoryCoordinationAction.LocalAction(actionFilters, transportService, clusterService, client) + ), + transportService.getTaskManager(), + localNode::getId, + transportService.getLocalNodeConnection(), + null + ); + // cluster utils publisher does not call AckListener, making some method calls hang indefinitely // in this test we have a single master node, and it acknowledges cluster state immediately final var publisher = ClusterServiceUtils.createClusterStatePublisher(clusterService.getClusterApplierService()); @@ -109,10 +126,10 @@ public void setUp() throws Exception { repositoriesService = new RepositoriesService( Settings.EMPTY, clusterService, - transportService, typesRegistry, typesRegistry, threadPool, + client, List.of() ); diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index e18e327734495..486390f27391c 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -8,9 +8,18 @@ package org.elasticsearch.repositories.blobstore; +import org.apache.logging.log4j.Level; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.admin.cluster.repositories.delete.DeleteRepositoryRequest; +import org.elasticsearch.action.admin.cluster.repositories.delete.TransportDeleteRepositoryAction; +import org.elasticsearch.action.admin.cluster.repositories.put.PutRepositoryRequest; +import org.elasticsearch.action.admin.cluster.repositories.put.TransportPutRepositoryAction; +import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.create.TransportCreateSnapshotAction; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.TransportGetSnapshotsAction; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.RefCountingListener; import org.elasticsearch.action.support.RefCountingRunnable; @@ -20,6 +29,7 @@ import org.elasticsearch.cluster.metadata.RepositoryMetadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Numbers; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; import org.elasticsearch.common.blobstore.OperationPurpose; import org.elasticsearch.common.bytes.BytesArray; @@ -49,6 +59,7 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.MockLog; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -535,4 +546,103 @@ public void testShardBlobsToDelete() { shardBlobsToDelete.getBlobPaths().forEachRemaining(s -> assertTrue(expectedBlobsToDelete.remove(s))); assertThat(expectedBlobsToDelete, empty()); } + + public void testUuidCreationLogging() { + final var repo = setupRepo(); + final var repoMetadata = repo.getMetadata(); + final var repoName = repoMetadata.name(); + final var snapshot = randomIdentifier(); + + MockLog.assertThatLogger( + () -> safeGet( + client().execute(TransportCreateSnapshotAction.TYPE, new CreateSnapshotRequest(repoName, snapshot).waitForCompletion(true)) + ), + BlobStoreRepository.class, + new MockLog.SeenEventExpectation( + "new repo uuid message", + BlobStoreRepository.class.getCanonicalName(), + Level.INFO, + Strings.format("Generated new repository UUID [*] for repository [%s] in generation [*]", repoName) + ) + ); + + MockLog.assertThatLogger( + // no more "Generated" messages ... + () -> { + safeGet(client().execute(TransportDeleteRepositoryAction.TYPE, new DeleteRepositoryRequest(repoName))); + + // we get a "Registering" message when re-registering the repository with ?verify=true (the default) + MockLog.assertThatLogger( + () -> safeGet( + client().execute( + TransportPutRepositoryAction.TYPE, + new PutRepositoryRequest(repoName).type("fs").verify(true).settings(repoMetadata.settings()) + ) + ), + RepositoriesService.class, + new MockLog.SeenEventExpectation( + "existing repo uuid message", + RepositoriesService.class.getCanonicalName(), + Level.INFO, + Strings.format("Registering repository [%s] with repository UUID *", repoName) + ) + ); + + safeGet( + client().execute( + TransportCreateSnapshotAction.TYPE, + new CreateSnapshotRequest(repoName, randomIdentifier()).waitForCompletion(true) + ) + ); + assertTrue( + safeGet(client().execute(TransportGetSnapshotsAction.TYPE, new GetSnapshotsRequest(repoName))).getSnapshots() + .stream() + .anyMatch(snapshotInfo -> snapshotInfo.snapshotId().getName().equals(snapshot)) + ); + + safeGet(client().execute(TransportDeleteRepositoryAction.TYPE, new DeleteRepositoryRequest(repoName))); + + // No "Registering" message with ?verify=false because we don't read the repo data yet + MockLog.assertThatLogger( + () -> safeGet( + client().execute( + TransportPutRepositoryAction.TYPE, + new PutRepositoryRequest(repoName).type("fs").verify(false).settings(repoMetadata.settings()) + ) + ), + RepositoriesService.class, + new MockLog.UnseenEventExpectation( + "existing repo uuid message", + RepositoriesService.class.getCanonicalName(), + Level.INFO, + "Registering repository*" + ) + ); + + // But we do get the "Registering" message the first time we read the repo + MockLog.assertThatLogger( + () -> safeGet( + client().execute( + TransportCreateSnapshotAction.TYPE, + new CreateSnapshotRequest(repoName, randomIdentifier()).waitForCompletion(true) + ) + ), + RepositoriesService.class, + new MockLog.SeenEventExpectation( + "existing repo uuid message", + RepositoriesService.class.getCanonicalName(), + Level.INFO, + Strings.format("Registering repository [%s] with repository UUID *", repoName) + ) + ); + }, + BlobStoreRepository.class, + new MockLog.UnseenEventExpectation( + "no repo uuid generated messages", + BlobStoreRepository.class.getCanonicalName(), + Level.INFO, + "Generated new repository UUID*" + ) + ); + } } diff --git a/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java b/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java index a095c4e6409ac..80c93e05b8bd5 100644 --- a/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/script/VectorScoreScriptUtilsTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.mapper.vectors.KnnDenseVectorScriptDocValuesTests; import org.elasticsearch.script.VectorScoreScriptUtils.CosineSimilarity; import org.elasticsearch.script.VectorScoreScriptUtils.DotProduct; +import org.elasticsearch.script.VectorScoreScriptUtils.Hamming; import org.elasticsearch.script.VectorScoreScriptUtils.L1Norm; import org.elasticsearch.script.VectorScoreScriptUtils.L2Norm; import org.elasticsearch.script.field.vectors.BinaryDenseVectorDocValuesField; @@ -112,6 +113,12 @@ public void testFloatVectorClassBindings() throws IOException { containsString("query vector has a different number of dimensions [2] than the document vectors [5]") ); + e = expectThrows(IllegalArgumentException.class, () -> new Hamming(scoreScript, queryVector, fieldName)); + assertThat(e.getMessage(), containsString("hamming distance is only supported for byte vectors")); + + e = expectThrows(IllegalArgumentException.class, () -> new Hamming(scoreScript, invalidQueryVector, fieldName)); + assertThat(e.getMessage(), containsString("hamming distance is only supported for byte vectors")); + // Check scripting infrastructure integration DotProduct dotProduct = new DotProduct(scoreScript, queryVector, fieldName); assertEquals(65425.6249, dotProduct.dotProduct(), 0.001); @@ -199,6 +206,11 @@ public void testByteVectorClassBindings() throws IOException { e.getMessage(), containsString("query vector has a different number of dimensions [2] than the document vectors [5]") ); + e = expectThrows(IllegalArgumentException.class, () -> new Hamming(scoreScript, invalidQueryVector, fieldName)); + assertThat( + e.getMessage(), + containsString("query vector has a different number of dimensions [2] than the document vectors [5]") + ); // Check scripting infrastructure integration assertEquals(17382.0, new DotProduct(scoreScript, queryVector, fieldName).dotProduct(), 0.001); @@ -207,6 +219,8 @@ public void testByteVectorClassBindings() throws IOException { assertEquals(135.0, new L1Norm(scoreScript, hexidecimalString, fieldName).l1norm(), 0.001); assertEquals(116.897, new L2Norm(scoreScript, queryVector, fieldName).l2norm(), 0.001); assertEquals(116.897, new L2Norm(scoreScript, hexidecimalString, fieldName).l2norm(), 0.001); + assertEquals(13.0, new Hamming(scoreScript, queryVector, fieldName).hamming(), 0.001); + assertEquals(13.0, new Hamming(scoreScript, hexidecimalString, fieldName).hamming(), 0.001); DotProduct dotProduct = new DotProduct(scoreScript, queryVector, fieldName); when(scoreScript._getDocId()).thenReturn(1); e = expectThrows(IllegalArgumentException.class, dotProduct::dotProduct); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java index 26b7945434c1b..e2bd711448c9d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ScriptedMetricAggregatorTests.java @@ -65,6 +65,12 @@ public class ScriptedMetricAggregatorTests extends AggregatorTestCase { "reduceScript", Collections.emptyMap() ); + private static final Script REDUCE_SCRIPT_COUNT_STATES = new Script( + ScriptType.INLINE, + MockScriptEngine.NAME, + "reduceScriptCountStates", + Collections.emptyMap() + ); private static final Script INIT_SCRIPT_SCORE = new Script( ScriptType.INLINE, @@ -170,6 +176,10 @@ public static void initMockScripts() { List states = (List) params.get("states"); return states.stream().filter(a -> a instanceof Number).map(a -> (Number) a).mapToInt(Number::intValue).sum(); }); + SCRIPTS.put("reduceScriptCountStates", params -> { + List states = (List) params.get("states"); + return states.size(); + }); SCRIPTS.put("initScriptScore", params -> { Map state = (Map) params.get("state"); @@ -308,7 +318,7 @@ public void testScriptedMetricWithoutCombine() throws IOException { IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { searchAndReduce(indexReader, new AggTestConfig(aggregationBuilder)); }); - assertEquals(exception.getMessage(), "[combineScript] must not be null: [scriptedMetric]"); + assertEquals(exception.getMessage(), "[combine_script] must not be null: [scriptedMetric]"); } } } @@ -327,7 +337,7 @@ public void testScriptedMetricWithoutReduce() throws IOException { IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> { searchAndReduce(indexReader, new AggTestConfig(aggregationBuilder)); }); - assertEquals(exception.getMessage(), "[reduceScript] must not be null: [scriptedMetric]"); + assertEquals(exception.getMessage(), "[reduce_script] must not be null: [scriptedMetric]"); } } } @@ -354,6 +364,31 @@ public void testScriptedMetricWithCombine() throws IOException { } } + public void testNoParallelization() throws IOException { + try (Directory directory = newDirectory()) { + int numDocs = randomInt(100); + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + for (int i = 0; i < numDocs; i++) { + indexWriter.addDocument(singleton(new SortedNumericDocValuesField("number", i))); + } + } + try (DirectoryReader indexReader = DirectoryReader.open(directory)) { + ScriptedMetricAggregationBuilder aggregationBuilder = new ScriptedMetricAggregationBuilder(AGG_NAME); + aggregationBuilder.initScript(INIT_SCRIPT) + .mapScript(MAP_SCRIPT) + .combineScript(COMBINE_SCRIPT) + .reduceScript(REDUCE_SCRIPT_COUNT_STATES); + ScriptedMetric scriptedMetric = searchAndReduce( + indexReader, + new AggTestConfig(aggregationBuilder).withSplitLeavesIntoSeperateAggregators(false) + ); + assertEquals(AGG_NAME, scriptedMetric.getName()); + assertNotNull(scriptedMetric.aggregation()); + assertEquals(1, scriptedMetric.aggregation()); + } + } + } + /** * test that uses the score of the documents */ diff --git a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java index a64283c8554b1..a7496e36955c3 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/InternalSnapshotsInfoServiceTests.java @@ -114,7 +114,7 @@ public void testSnapshotShardSizes() throws Exception { final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( Settings.builder().put(INTERNAL_SNAPSHOT_INFO_MAX_CONCURRENT_FETCHES_SETTING.getKey(), maxConcurrentFetches).build(), clusterService, - () -> repositoriesService, + repositoriesService, () -> rerouteService ); @@ -185,7 +185,7 @@ public void testErroneousSnapshotShardSizes() throws Exception { final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( Settings.builder().put(INTERNAL_SNAPSHOT_INFO_MAX_CONCURRENT_FETCHES_SETTING.getKey(), randomIntBetween(1, 10)).build(), clusterService, - () -> repositoriesService, + repositoriesService, () -> rerouteService ); @@ -274,7 +274,7 @@ public void testNoLongerMaster() throws Exception { final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( Settings.EMPTY, clusterService, - () -> repositoriesService, + repositoriesService, () -> rerouteService ); @@ -329,7 +329,7 @@ public IndexShardSnapshotStatus.Copy getShardSnapshotStatus(SnapshotId snapshotI final InternalSnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( Settings.EMPTY, clusterService, - () -> repositoriesService, + repositoriesService, () -> rerouteService ); diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index add40a1993e86..f4aa44f143c40 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -168,6 +168,8 @@ import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.repositories.RepositoryData; +import org.elasticsearch.repositories.VerifyNodeRepositoryAction; +import org.elasticsearch.repositories.VerifyNodeRepositoryCoordinationAction; import org.elasticsearch.repositories.blobstore.BlobStoreRepository; import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.repositories.fs.FsRepository; @@ -2051,6 +2053,7 @@ private final class TestClusterNode { threadPool = deterministicTaskQueue.getThreadPool(runnable -> DeterministicTaskQueue.onNodeLog(node, runnable)); masterService = new FakeThreadPoolMasterService(node.getName(), threadPool, deterministicTaskQueue::scheduleNow); final Settings settings = environment.settings(); + client = new NodeClient(settings, threadPool); final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); clusterService = new ClusterService( settings, @@ -2142,13 +2145,13 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { repositoriesService = new RepositoriesService( settings, clusterService, - transportService, Collections.singletonMap( FsRepository.TYPE, metadata -> new FsRepository(metadata, environment, xContentRegistry(), clusterService, bigArrays, recoverySettings) ), emptyMap(), threadPool, + client, List.of() ); final ActionFilters actionFilters = new ActionFilters(emptySet()); @@ -2165,12 +2168,12 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { nodeEnv = new NodeEnvironment(settings, environment); final NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(Collections.emptyList()); final ScriptService scriptService = new ScriptService(settings, emptyMap(), emptyMap(), () -> 1L); - client = new NodeClient(settings, threadPool); + final SetOnce rerouteServiceSetOnce = new SetOnce<>(); final SnapshotsInfoService snapshotsInfoService = new InternalSnapshotsInfoService( settings, clusterService, - () -> repositoriesService, + repositoriesService, rerouteServiceSetOnce::get ); allocationService = ESAllocationTestCase.createAllocationService( @@ -2253,6 +2256,20 @@ public RecyclerBytesStreamOutput newNetworkBytesStream() { actionFilters ) ); + actions.put( + VerifyNodeRepositoryAction.TYPE, + new VerifyNodeRepositoryAction.TransportAction( + transportService, + actionFilters, + threadPool, + clusterService, + repositoriesService + ) + ); + actions.put( + VerifyNodeRepositoryCoordinationAction.TYPE, + new VerifyNodeRepositoryCoordinationAction.LocalAction(actionFilters, transportService, clusterService, client) + ); final MetadataMappingService metadataMappingService = new MetadataMappingService(clusterService, indicesService); peerRecoverySourceService = new PeerRecoverySourceService( diff --git a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java index c2dfd70c22bd9..b0f73d0726002 100644 --- a/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java +++ b/server/src/test/java/org/elasticsearch/tasks/BanFailureLoggingTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.test.transport.StubbableTransport; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.AbstractSimpleTransportTestCase; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.NodeDisconnectedException; import org.elasticsearch.transport.TransportException; import org.elasticsearch.transport.TransportRequest; @@ -131,7 +132,7 @@ private void runTest( childTransportService.registerRequestHandler( "internal:testAction[c]", threadPool.executor(ThreadPool.Names.MANAGEMENT), // busy-wait for cancellation but not on a transport thread - (StreamInput in) -> new TransportRequest.Empty(in) { + (StreamInput in) -> new TransportRequest(in) { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { return new CancellableTask(id, type, action, "", parentTaskId, headers); @@ -163,7 +164,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, parentTransportService.sendChildRequest( childTransportService.getLocalDiscoNode(), "internal:testAction[c]", - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), parentTask, TransportRequestOptions.EMPTY, new ChildResponseHandler(() -> parentTransportService.getTaskManager().unregister(parentTask)) diff --git a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java index 0f4a60665b35a..bc13d3fde7e31 100644 --- a/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java +++ b/server/src/test/java/org/elasticsearch/transport/ClusterConnectionManagerTests.java @@ -309,7 +309,7 @@ public void testConcurrentConnects() throws Exception { if (nodeConnectedCount.get() == 0) { // Any successful connections were closed - assertTrue(pendingCloses.tryAcquire(threadCount, 10, TimeUnit.SECONDS)); + safeAcquire(threadCount, pendingCloses); pendingCloses.release(threadCount); assertTrue(connections.stream().allMatch(Transport.Connection::isClosed)); assertEquals(0, connectionManager.size()); @@ -320,7 +320,7 @@ public void testConcurrentConnects() throws Exception { if (randomBoolean()) { Releasables.close(releasables); - assertTrue(pendingCloses.tryAcquire(threadCount, 10, TimeUnit.SECONDS)); + safeAcquire(threadCount, pendingCloses); pendingCloses.release(threadCount); assertEquals(0, connectionManager.size()); assertTrue(connections.stream().allMatch(Transport.Connection::isClosed)); diff --git a/server/src/test/java/org/elasticsearch/transport/LeakTrackerTests.java b/server/src/test/java/org/elasticsearch/transport/LeakTrackerTests.java new file mode 100644 index 0000000000000..f13858351b7ee --- /dev/null +++ b/server/src/test/java/org/elasticsearch/transport/LeakTrackerTests.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.transport; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.elasticsearch.core.AbstractRefCounted; +import org.elasticsearch.core.Assertions; +import org.elasticsearch.core.RefCounted; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.ReachabilityChecker; +import org.junit.Before; + +import java.io.Closeable; +import java.io.IOException; +import java.util.stream.Stream; + +public class LeakTrackerTests extends ESTestCase { + + private static final Logger logger = LogManager.getLogger(); + + private final TrackedObjectLifecycle trackedObjectLifecycle; + private ReachabilityChecker reachabilityChecker; + + @Before + public void createReachabilityTracker() { + reachabilityChecker = new ReachabilityChecker(); + } + + @Before + public void onlyRunWhenAssertionsAreEnabled() { + assumeTrue("Many of these tests don't make sense when assertions are disabled", Assertions.ENABLED); + } + + public LeakTrackerTests(@Name("trackingMethod") TrackedObjectLifecycle trackedObjectLifecycle) { + this.trackedObjectLifecycle = trackedObjectLifecycle; + } + + @ParametersFactory(shuffle = false) + public static Iterable parameters() { + return Stream.of( + new PojoTrackedObjectLifecycle(), + new ReleasableTrackedObjectLifecycle(), + new ReferenceCountedTrackedObjectLifecycle() + ).map(i -> new Object[] { i }).toList(); + } + + @SuppressWarnings("resource") + public void testWillLogErrorWhenTrackedObjectIsNotClosed() throws Exception { + // Let it go out of scope without closing + trackedObjectLifecycle.createAndTrack(reachabilityChecker); + reachabilityChecker.ensureUnreachable(); + assertBusy(ESTestCase::assertLeakDetected); + } + + public void testWillNotLogErrorWhenTrackedObjectIsClosed() throws IOException { + // Close before letting it go out of scope + trackedObjectLifecycle.createAndTrack(reachabilityChecker).close(); + reachabilityChecker.ensureUnreachable(); + } + + /** + * Encapsulates the lifecycle for a particular type of tracked object + */ + public interface TrackedObjectLifecycle { + + /** + * Create the tracked object, implementations must + * - track it with the {@link LeakTracker} + * - register it with the passed reachability checker + * @param reachabilityChecker The reachability checker + * @return A {@link Closeable} that retains a reference to the tracked object, and when closed will do the appropriate cleanup + */ + Closeable createAndTrack(ReachabilityChecker reachabilityChecker); + } + + private static class PojoTrackedObjectLifecycle implements TrackedObjectLifecycle { + + @Override + public Closeable createAndTrack(ReachabilityChecker reachabilityChecker) { + final Object object = reachabilityChecker.register(new Object()); + final LeakTracker.Leak leak = LeakTracker.INSTANCE.track(object); + return () -> { + logger.info("This log line retains a reference to {}", object); + leak.close(); + }; + } + + @Override + public String toString() { + return "LeakTracker.INSTANCE.track(Object)"; + } + } + + private static class ReferenceCountedTrackedObjectLifecycle implements TrackedObjectLifecycle { + + @Override + public Closeable createAndTrack(ReachabilityChecker reachabilityChecker) { + RefCounted refCounted = LeakTracker.wrap(reachabilityChecker.register((RefCounted) AbstractRefCounted.of(() -> {}))); + refCounted.incRef(); + refCounted.tryIncRef(); + return () -> { + refCounted.decRef(); // tryIncRef + refCounted.decRef(); // incRef + refCounted.decRef(); // implicit + }; + } + + @Override + public String toString() { + return "LeakTracker.wrap(RefCounted)"; + } + } + + private static class ReleasableTrackedObjectLifecycle implements TrackedObjectLifecycle { + + @Override + public Closeable createAndTrack(ReachabilityChecker reachabilityChecker) { + return LeakTracker.wrap(reachabilityChecker.register(Releasables.assertOnce(() -> {}))); + } + + @Override + public String toString() { + return "LeakTracker.wrap(Releasable)"; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 8297070ed3d5f..77a57bf1110fb 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -761,7 +761,7 @@ public void testNoChannelsExceptREG() throws Exception { .sendRequest( randomNonNegativeLong(), "arbitrary", - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), TransportRequestOptions.of(null, type) ) ).getMessage(), diff --git a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java index 3346bd40aec5a..2fef1d572fc64 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportActionProxyTests.java @@ -500,8 +500,8 @@ public void testIsProxyAction() { } public void testIsProxyRequest() { - assertTrue(TransportActionProxy.isProxyRequest(new TransportActionProxy.ProxyRequest<>(TransportRequest.Empty.INSTANCE, null))); - assertFalse(TransportActionProxy.isProxyRequest(TransportRequest.Empty.INSTANCE)); + assertTrue(TransportActionProxy.isProxyRequest(new TransportActionProxy.ProxyRequest<>(new EmptyRequest(), null))); + assertFalse(TransportActionProxy.isProxyRequest(new EmptyRequest())); } static class CapturingTransportChannel implements TransportChannel { diff --git a/server/src/test/java/org/elasticsearch/transport/TransportLoggerTests.java b/server/src/test/java/org/elasticsearch/transport/TransportLoggerTests.java index 679432f8b60a0..ce8efc88b090a 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportLoggerTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportLoggerTests.java @@ -9,8 +9,6 @@ import org.apache.logging.log4j.Level; import org.elasticsearch.TransportVersion; -import org.elasticsearch.action.admin.cluster.stats.ClusterStatsRequest; -import org.elasticsearch.action.admin.cluster.stats.TransportClusterStatsAction; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput; import org.elasticsearch.common.settings.Settings; @@ -33,7 +31,7 @@ public void testLoggingHandler() throws IOException { + ", type: request" + ", version: .*" + ", header size: \\d+B" - + ", action: cluster:monitor/stats]" + + ", action: internal:test]" + " WRITE: \\d+B"; final MockLog.LoggingExpectation writeExpectation = new MockLog.PatternSeenEventExpectation( "hot threads request", @@ -47,11 +45,11 @@ public void testLoggingHandler() throws IOException { + ", type: request" + ", version: .*" + ", header size: \\d+B" - + ", action: cluster:monitor/stats]" + + ", action: internal:test]" + " READ: \\d+B"; final MockLog.LoggingExpectation readExpectation = new MockLog.PatternSeenEventExpectation( - "cluster monitor request", + "cluster state request", TransportLogger.class.getCanonicalName(), Level.TRACE, readPattern @@ -73,9 +71,9 @@ private BytesReference buildRequest() throws IOException { try (RecyclerBytesStreamOutput bytesStreamOutput = new RecyclerBytesStreamOutput(recycler)) { OutboundMessage.Request request = new OutboundMessage.Request( new ThreadContext(Settings.EMPTY), - new ClusterStatsRequest(), + new EmptyRequest(), TransportVersion.current(), - TransportClusterStatsAction.TYPE.name(), + "internal:test", randomInt(30), false, compress diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java index a3b44c702e692..fef835ce78f64 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceDeserializationFailureTests.java @@ -69,7 +69,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req transportService.registerRequestHandler( testActionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, - TransportRequest.Empty::new, + EmptyRequest::new, (request, channel, task) -> channel.sendResponse(TransportResponse.Empty.INSTANCE) ); @@ -86,7 +86,7 @@ protected void onSendRequest(long requestId, String action, TransportRequest req transportService.sendRequest( otherNode, testActionName, - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), TransportRequestOptions.EMPTY, new TransportResponseHandler() { @Override @@ -151,7 +151,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, transportService.sendChildRequest( otherNode, testActionName, - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), parentTask, TransportRequestOptions.EMPTY, new TransportResponseHandler() { diff --git a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java index 7b3de5f142597..b6cfba8a4e38a 100644 --- a/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java +++ b/server/src/test/java/org/elasticsearch/transport/TransportServiceLifecycleTests.java @@ -77,7 +77,7 @@ public void testHandlersCompleteAtShutdown() throws Exception { nodeB.transportService.sendRequest( randomFrom(random, nodeA, nodeB).transportService.getLocalNode(), TestNode.randomActionName(random), - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), new TransportResponseHandler() { final AtomicBoolean completed = new AtomicBoolean(); @@ -120,7 +120,7 @@ public Executor executor() { // every handler is completed even if the request or response are being handled concurrently with shutdown keepGoing.set(false); - assertTrue(requestPermits.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS)); + safeAcquire(Integer.MAX_VALUE, requestPermits); for (final var thread : threads) { thread.join(); } @@ -135,7 +135,7 @@ public void testInternalSendExceptionForksToHandlerExecutor() { nodeA.transportService.sendRequest( nodeA.getThrowingConnection(), TestNode.randomActionName(random()), - new TransportRequest.Empty(), + new EmptyRequest(), TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(future, unusedReader(), deterministicTaskQueue::scheduleNow) ); @@ -154,7 +154,7 @@ public void testInternalSendExceptionForksToGenericIfHandlerDoesNotFork() { nodeA.transportService.sendRequest( nodeA.getThrowingConnection(), TestNode.randomActionName(random()), - new TransportRequest.Empty(), + new EmptyRequest(), TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(future.delegateResponse((l, e) -> { assertThat(Thread.currentThread().getName(), containsString("[" + ThreadPool.Names.GENERIC + "]")); @@ -183,7 +183,7 @@ public void testInternalSendExceptionForcesExecutionOnHandlerExecutor() { nodeA.transportService.sendRequest( nodeA.getThrowingConnection(), TestNode.randomActionName(random()), - new TransportRequest.Empty(), + new EmptyRequest(), TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(future.delegateResponse((l, e) -> { assertThat(Thread.currentThread().getName(), containsString("[" + Executors.FIXED_BOUNDED_QUEUE + "]")); @@ -209,7 +209,7 @@ public void testInternalSendExceptionCompletesHandlerOnCallingThreadIfTransportS nodeA.transportService.sendRequest( nodeA.getThrowingConnection(), TestNode.randomActionName(random()), - new TransportRequest.Empty(), + new EmptyRequest(), TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>(future.delegateResponse((l, e) -> { assertSame(testThread, Thread.currentThread()); @@ -256,7 +256,7 @@ private void onConnectionClosedUsesHandlerExecutor(Settings settings, String exe nodeA.transportService.sendRequest( connection, TestNode.randomActionName(random()), - new TransportRequest.Empty(), + new EmptyRequest(), TransportRequestOptions.EMPTY, new ActionListenerResponseHandler<>( ActionListener.assertOnce(ActionTestUtils.assertNoSuccessListener(future::onResponse).delegateResponse((l, e) -> { @@ -361,7 +361,7 @@ public ExecutorService executor(String name) { transportService.registerRequestHandler( ACTION_NAME_PREFIX + executorName, getExecutor(executorName), - TransportRequest.Empty::new, + EmptyRequest::new, (request, channel, task) -> { if (randomBoolean()) { channel.sendResponse(TransportResponse.Empty.INSTANCE); diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle index c671d58e1e395..733bfd1d4bd29 100644 --- a/test/fixtures/krb5kdc-fixture/build.gradle +++ b/test/fixtures/krb5kdc-fixture/build.gradle @@ -22,7 +22,7 @@ dockerFixtures { configurations { all { - transitive = false + exclude group: 'org.hamcrest', module: 'hamcrest-core' } krb5ConfHdfsFile { canBeConsumed = true @@ -36,21 +36,18 @@ configurations { dependencies { testImplementation project(':test:framework') - api "junit:junit:${versions.junit}" api project(':test:fixtures:testcontainer-utils') - api "org.testcontainers:testcontainers:${versions.testcontainer}" - implementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}" + api("org.testcontainers:testcontainers:${versions.testcontainer}") { + transitive = false + } + implementation("com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"){ + transitive = false + } implementation "org.slf4j:slf4j-api:${versions.slf4j}" - implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" + // implementation "com.github.docker-java:docker-java-api:${versions.dockerJava}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" - runtimeOnly "com.github.docker-java:docker-java-transport-zerodep:${versions.dockerJava}" - runtimeOnly "com.github.docker-java:docker-java-transport:${versions.dockerJava}" - runtimeOnly "com.github.docker-java:docker-java-core:${versions.dockerJava}" - runtimeOnly "org.apache.commons:commons-compress:${versions.commonsCompress}" - runtimeOnly "org.rnorth.duct-tape:duct-tape:${versions.ductTape}" - // ensure we have proper logging during when used in tests runtimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}" runtimeOnly "org.hamcrest:hamcrest:${versions.hamcrest}" diff --git a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java index 30623c6bafd6b..8ef80c08517de 100644 --- a/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java +++ b/test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java @@ -223,7 +223,7 @@ static Map getCodebases() { addClassCodebase(codebases, "elasticsearch-core", "org.elasticsearch.core.Booleans"); addClassCodebase(codebases, "elasticsearch-cli", "org.elasticsearch.cli.Command"); addClassCodebase(codebases, "elasticsearch-preallocate", "org.elasticsearch.preallocate.Preallocate"); - addClassCodebase(codebases, "elasticsearch-vec", "org.elasticsearch.vec.VectorScorerFactory"); + addClassCodebase(codebases, "elasticsearch-simdvec", "org.elasticsearch.simdvec.VectorScorerFactory"); addClassCodebase(codebases, "framework", "org.elasticsearch.test.ESTestCase"); return codebases; } diff --git a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java index b1eddf927d3f3..9378b51de78df 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java +++ b/test/framework/src/main/java/org/elasticsearch/common/util/MockBigArrays.java @@ -389,8 +389,8 @@ public byte get(long index) { } @Override - public byte set(long index, byte value) { - return in.set(index, value); + public void set(long index, byte value) { + in.set(index, value); } @Override @@ -469,8 +469,13 @@ public int get(long index) { } @Override - public int set(long index, int value) { - return in.set(index, value); + public int getAndSet(long index, int value) { + return in.getAndSet(index, value); + } + + @Override + public void set(long index, int value) { + in.set(index, value); } @Override @@ -524,8 +529,13 @@ public long get(long index) { } @Override - public long set(long index, long value) { - return in.set(index, value); + public long getAndSet(long index, long value) { + return in.getAndSet(index, value); + } + + @Override + public void set(long index, long value) { + in.set(index, value); } @Override @@ -584,8 +594,8 @@ public float get(long index) { } @Override - public float set(long index, float value) { - return in.set(index, value); + public void set(long index, float value) { + in.set(index, value); } @Override @@ -629,8 +639,8 @@ public double get(long index) { } @Override - public double set(long index, double value) { - return in.set(index, value); + public void set(long index, double value) { + in.set(index, value); } @Override diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 3a7a31e761e7f..1c7cabb541581 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -268,7 +268,8 @@ public static EngineConfig copy(EngineConfig config, LongSupplier globalCheckpoi config.getLeafSorter(), config.getRelativeTimeInNanosSupplier(), config.getIndexCommitListener(), - config.isPromotableToPrimary() + config.isPromotableToPrimary(), + config.getMapperService() ); } @@ -299,7 +300,8 @@ public EngineConfig copy(EngineConfig config, Analyzer analyzer) { config.getLeafSorter(), config.getRelativeTimeInNanosSupplier(), config.getIndexCommitListener(), - config.isPromotableToPrimary() + config.isPromotableToPrimary(), + config.getMapperService() ); } @@ -330,7 +332,8 @@ public EngineConfig copy(EngineConfig config, MergePolicy mergePolicy) { config.getLeafSorter(), config.getRelativeTimeInNanosSupplier(), config.getIndexCommitListener(), - config.isPromotableToPrimary() + config.isPromotableToPrimary(), + config.getMapperService() ); } @@ -854,7 +857,8 @@ public EngineConfig config( null, this::relativeTimeInNanos, indexCommitListener, - true + true, + null ); } @@ -893,7 +897,8 @@ protected EngineConfig config(EngineConfig config, Store store, Path translogPat config.getLeafSorter(), config.getRelativeTimeInNanosSupplier(), config.getIndexCommitListener(), - config.isPromotableToPrimary() + config.isPromotableToPrimary(), + config.getMapperService() ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 50436ad64c8af..0486022620398 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; @@ -34,6 +35,7 @@ import org.elasticsearch.common.util.MockPageCacheRecycler; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -773,12 +775,14 @@ protected RandomIndexWriter indexWriterForSyntheticSource(Directory directory) t protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = indexWriterForSyntheticSource(directory); - LuceneDocument doc = mapper.parse(source(build)).rootDoc(); - iw.addDocument(doc); + ParsedDocument doc = mapper.parse(source(build)); + doc.updateSeqID(0, 0); + doc.version().setLongValue(0); + iw.addDocuments(doc.docs()); iw.close(); - try (DirectoryReader reader = DirectoryReader.open(directory)) { - String syntheticSource = syntheticSource(mapper, reader, 0); - roundTripSyntheticSource(mapper, syntheticSource, reader); + try (DirectoryReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) { + String syntheticSource = syntheticSource(mapper, indexReader, doc.docs().size() - 1); + roundTripSyntheticSource(mapper, syntheticSource, indexReader); return syntheticSource; } } @@ -797,10 +801,14 @@ protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer { @@ -1239,7 +1238,7 @@ public final void testNoSyntheticSourceForScript() throws IOException { } public final void testSyntheticSourceInObject() throws IOException { - boolean ignoreMalformed = supportsIgnoreMalformed() ? rarely() : false; + boolean ignoreMalformed = shouldUseIgnoreMalformed(); SyntheticSourceExample syntheticSourceExample = syntheticSourceSupport(ignoreMalformed).example(5); DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { b.startObject("obj").startObject("properties").startObject("field"); @@ -1255,7 +1254,7 @@ public final void testSyntheticSourceInObject() throws IOException { public final void testSyntheticEmptyList() throws IOException { assumeTrue("Field does not support [] as input", supportsEmptyInputArray()); - boolean ignoreMalformed = supportsIgnoreMalformed() ? rarely() : false; + boolean ignoreMalformed = shouldUseIgnoreMalformed(); SyntheticSourceSupport support = syntheticSourceSupport(ignoreMalformed); SyntheticSourceExample syntheticSourceExample = support.example(5); DocumentMapper mapper = createDocumentMapper(syntheticSourceMapping(b -> { @@ -1268,6 +1267,11 @@ public final void testSyntheticEmptyList() throws IOException { assertThat(syntheticSource(mapper, b -> b.startArray("field").endArray()), equalTo(expected)); } + private boolean shouldUseIgnoreMalformed() { + // 5% of test runs use ignore_malformed + return supportsIgnoreMalformed() && randomDouble() <= 0.05; + } + public final void testSyntheticEmptyListNoDocValuesLoader() throws IOException { assumeTrue("Field does not support [] as input", supportsEmptyInputArray()); assertNoDocValueLoader(b -> b.startArray("field").endArray()); @@ -1490,7 +1494,7 @@ private void assertNoDocValueLoader(CheckedConsumer examples = new ArrayList<>(syntheticSourceSupport(ignoreMalformed).invalidExample()); if (supportsCopyTo()) { examples.add( diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 6920083f2a1a6..0d20c613b27a8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -788,6 +788,17 @@ protected static void checkStaticState() throws Exception { } } + /** + * Assert that at least one leak was detected, also clear the list of detected leaks + * so the test won't fail for leaks detected up until this point. + */ + protected static void assertLeakDetected() { + synchronized (loggedLeaks) { + assertFalse("No leaks have been detected", loggedLeaks.isEmpty()); + loggedLeaks.clear(); + } + } + // this must be a separate method from other ensure checks above so suite scoped integ tests can call...TODO: fix that public final void ensureAllSearchContextsReleased() throws Exception { assertBusy(() -> MockSearchService.assertNoInFlightContext()); @@ -990,6 +1001,35 @@ public static float randomFloat() { return random().nextFloat(); } + /** + * Returns a float value in the interval [start, end) if lowerInclusive is + * set to true, (start, end) otherwise. + * + * @param start lower bound of interval to draw uniformly distributed random numbers from + * @param end upper bound + * @param lowerInclusive whether or not to include lower end of the interval + */ + public static float randomFloatBetween(float start, float end, boolean lowerInclusive) { + float result; + + if (start == -Float.MAX_VALUE || end == Float.MAX_VALUE) { + // formula below does not work with very large floats + result = Float.intBitsToFloat(randomInt()); + while (result < start || result > end || Double.isNaN(result)) { + result = Float.intBitsToFloat(randomInt()); + } + } else { + result = randomFloat(); + if (lowerInclusive == false) { + while (result <= 0.0f) { + result = randomFloat(); + } + } + result = result * end + (1.0f - result) * start; + } + return result; + } + public static double randomDouble() { return random().nextDouble(); } @@ -2195,14 +2235,18 @@ public static void safeAwait(CountDownLatch countDownLatch) { } public static void safeAcquire(Semaphore semaphore) { + safeAcquire(1, semaphore); + } + + public static void safeAcquire(int permits, Semaphore semaphore) { try { assertTrue( "safeAcquire: Semaphore did not acquire permit within the timeout", - semaphore.tryAcquire(SAFE_AWAIT_TIMEOUT.millis(), TimeUnit.MILLISECONDS) + semaphore.tryAcquire(permits, SAFE_AWAIT_TIMEOUT.millis(), TimeUnit.MILLISECONDS) ); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - fail(e, "safeAcquire: interrupted waiting for Semaphore to acquire permit"); + fail(e, "safeAcquire: interrupted waiting for Semaphore to acquire " + permits + " permit(s)"); } } diff --git a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java index b6a8bc343687f..c0ce4061f2459 100644 --- a/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/transport/AbstractSimpleTransportTestCase.java @@ -514,7 +514,7 @@ public Executor executor() { } public void testMessageListeners() throws Exception { - final TransportRequestHandler requestHandler = (request, channel, task) -> { + final TransportRequestHandler requestHandler = (request, channel, task) -> { try { if (randomBoolean()) { channel.sendResponse(TransportResponse.Empty.INSTANCE); @@ -527,8 +527,8 @@ public void testMessageListeners() throws Exception { } }; final String ACTION = "internal:action"; - serviceA.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); - serviceB.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), TransportRequest.Empty::new, requestHandler); + serviceA.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), EmptyRequest::new, requestHandler); + serviceB.registerRequestHandler(ACTION, threadPool.executor(ThreadPool.Names.GENERIC), EmptyRequest::new, requestHandler); class CountingListener implements TransportMessageListener { AtomicInteger requestsReceived = new AtomicInteger(); @@ -585,7 +585,7 @@ public void onRequestSent( serviceB.addMessageListener(tracerB); try { - submitRequest(serviceA, nodeB, ACTION, TransportRequest.Empty.INSTANCE, NOOP_HANDLER).get(); + submitRequest(serviceA, nodeB, ACTION, new EmptyRequest(), NOOP_HANDLER).get(); } catch (ExecutionException e) { assertThat(e.getCause(), instanceOf(ElasticsearchException.class)); assertThat(ExceptionsHelper.unwrapCause(e.getCause()).getMessage(), equalTo("simulated")); @@ -604,7 +604,7 @@ public void onRequestSent( }); try { - submitRequest(serviceB, nodeA, ACTION, TransportRequest.Empty.INSTANCE, NOOP_HANDLER).get(); + submitRequest(serviceB, nodeA, ACTION, new EmptyRequest(), NOOP_HANDLER).get(); } catch (ExecutionException e) { assertThat(e.getCause(), instanceOf(ElasticsearchException.class)); assertThat(ExceptionsHelper.unwrapCause(e.getCause()).getMessage(), equalTo("simulated")); @@ -624,7 +624,7 @@ public void onRequestSent( // use assert busy as callbacks are called on a different thread try { - submitRequest(serviceA, nodeA, ACTION, TransportRequest.Empty.INSTANCE, NOOP_HANDLER).get(); + submitRequest(serviceA, nodeA, ACTION, new EmptyRequest(), NOOP_HANDLER).get(); } catch (ExecutionException e) { assertThat(e.getCause(), instanceOf(ElasticsearchException.class)); assertThat(ExceptionsHelper.unwrapCause(e.getCause()).getMessage(), equalTo("simulated")); @@ -648,7 +648,7 @@ public void testVoidMessageCompressed() throws Exception { serviceA.registerRequestHandler( "internal:sayHello", threadPool.executor(ThreadPool.Names.GENERIC), - TransportRequest.Empty::new, + EmptyRequest::new, (request, channel, task) -> { try { channel.sendResponse(TransportResponse.Empty.INSTANCE); @@ -673,7 +673,7 @@ public void testVoidMessageCompressed() throws Exception { serviceC, nodeA, "internal:sayHello", - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), new TransportResponseHandler<>() { @Override public TransportResponse.Empty read(StreamInput in) { @@ -1257,7 +1257,7 @@ public void handleException(TransportException exp) { } waitForever.countDown(); doneWaitingForever.await(); - assertTrue(inFlight.tryAcquire(Integer.MAX_VALUE, 10, TimeUnit.SECONDS)); + safeAcquire(Integer.MAX_VALUE, inFlight); } @TestLogging( @@ -3185,48 +3185,38 @@ public void testFailToSendIllegalStateException() throws InterruptedException { public void testChannelToString() { final String ACTION = "internal:action"; - serviceA.registerRequestHandler( - ACTION, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TransportRequest.Empty::new, - (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("DirectResponseChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}') - ) - ); - assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - ); - serviceB.registerRequestHandler( - ACTION, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TransportRequest.Empty::new, - (request, channel, task) -> { - assertThat( - channel.toString(), - allOf( - containsString("TcpTransportChannel"), - containsString('{' + ACTION + '}'), - containsString("TaskTransportChannel{task=" + task.getId() + '}'), - containsString("localAddress="), - containsString(serviceB.getLocalNode().getAddress().toString()) - ) - ); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - ); + serviceA.registerRequestHandler(ACTION, EsExecutors.DIRECT_EXECUTOR_SERVICE, EmptyRequest::new, (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("DirectResponseChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}') + ) + ); + assertThat(new ChannelActionListener<>(channel).toString(), containsString(channel.toString())); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + }); + serviceB.registerRequestHandler(ACTION, EsExecutors.DIRECT_EXECUTOR_SERVICE, EmptyRequest::new, (request, channel, task) -> { + assertThat( + channel.toString(), + allOf( + containsString("TcpTransportChannel"), + containsString('{' + ACTION + '}'), + containsString("TaskTransportChannel{task=" + task.getId() + '}'), + containsString("localAddress="), + containsString(serviceB.getLocalNode().getAddress().toString()) + ) + ); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + }); PlainActionFuture.get( f -> submitRequest( serviceA, serviceA.getLocalNode(), ACTION, - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), new ActionListenerResponseHandler<>( f, ignored -> TransportResponse.Empty.INSTANCE, @@ -3242,7 +3232,7 @@ public void testChannelToString() { serviceA, serviceB.getLocalNode(), ACTION, - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), new ActionListenerResponseHandler<>( f, ignored -> TransportResponse.Empty.INSTANCE, @@ -3360,23 +3350,18 @@ public void testWatchdogLogging() { final var barrier = new CyclicBarrier(2); final var threadNameFuture = new PlainActionFuture(); final var actionName = "internal:action"; - serviceA.registerRequestHandler( - actionName, - EsExecutors.DIRECT_EXECUTOR_SERVICE, - TransportRequest.Empty::new, - (request, channel, task) -> { - threadNameFuture.onResponse(Thread.currentThread().getName()); - safeAwait(barrier); - channel.sendResponse(TransportResponse.Empty.INSTANCE); - } - ); + serviceA.registerRequestHandler(actionName, EsExecutors.DIRECT_EXECUTOR_SERVICE, EmptyRequest::new, (request, channel, task) -> { + threadNameFuture.onResponse(Thread.currentThread().getName()); + safeAwait(barrier); + channel.sendResponse(TransportResponse.Empty.INSTANCE); + }); final var responseLatch = new CountDownLatch(1); submitRequest( serviceB, nodeA, actionName, - new TransportRequest.Empty(), + new EmptyRequest(), new ActionListenerResponseHandler( ActionTestUtils.assertNoFailureListener(t -> responseLatch.countDown()), in -> TransportResponse.Empty.INSTANCE, @@ -3470,7 +3455,7 @@ public void onFailure(final Exception e) { serviceC.sendRequest( connection, "fail-to-send-action", - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), TransportRequestOptions.EMPTY, new TransportResponseHandler.Empty() { @Override diff --git a/test/framework/src/main/java/org/elasticsearch/transport/EmptyRequest.java b/test/framework/src/main/java/org/elasticsearch/transport/EmptyRequest.java new file mode 100644 index 0000000000000..6b04789eec059 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/transport/EmptyRequest.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.transport; + +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * A transport request with an empty payload. Not really entirely empty: all transport requests include the parent task ID, a request ID, + * and the remote address (if applicable). + */ +public final class EmptyRequest extends TransportRequest { + public EmptyRequest() {} + + public EmptyRequest(StreamInput in) throws IOException { + super(in); + } +} diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index cdcc0e495582a..9fe7fc647455e 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -29,9 +29,11 @@ import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.LeafHistogramFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.SourceLoader; @@ -44,6 +46,7 @@ import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.xcontent.CopyingXContentParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -287,8 +290,12 @@ protected boolean supportsParsingObject() { @Override public void parse(DocumentParserContext context) throws IOException { context.path().add(simpleName()); + + boolean shouldStoreMalformedDataForSyntheticSource = context.mappingLookup().isSourceSynthetic() && ignoreMalformed(); XContentParser.Token token; XContentSubParser subParser = null; + XContentBuilder malformedDataForSyntheticSource = null; + try { token = context.parser().currentToken(); if (token == XContentParser.Token.VALUE_NULL) { @@ -299,10 +306,16 @@ public void parse(DocumentParserContext context) throws IOException { ArrayList counts = null; // should be an object ensureExpectedToken(XContentParser.Token.START_OBJECT, token, context.parser()); - subParser = new XContentSubParser(context.parser()); + if (shouldStoreMalformedDataForSyntheticSource) { + var copyingParser = new CopyingXContentParser(context.parser()); + malformedDataForSyntheticSource = copyingParser.getBuilder(); + subParser = new XContentSubParser(copyingParser); + } else { + subParser = new XContentSubParser(context.parser()); + } token = subParser.nextToken(); while (token != XContentParser.Token.END_OBJECT) { - // should be an field + // should be a field ensureExpectedToken(XContentParser.Token.FIELD_NAME, token, subParser); String fieldName = subParser.currentName(); if (fieldName.equals(VALUES_FIELD.getPreferredName())) { @@ -427,7 +440,17 @@ public void parse(DocumentParserContext context) throws IOException { if (subParser != null) { // close the subParser so we advance to the end of the object subParser.close(); + } else if (shouldStoreMalformedDataForSyntheticSource) { + // We have a malformed value, but it's not an object given that `subParser` is null. + // So we just remember whatever it is. + malformedDataForSyntheticSource = XContentBuilder.builder(context.parser().contentType().xContent()) + .copyCurrentStructure(context.parser()); + } + + if (malformedDataForSyntheticSource != null) { + context.doc().add(IgnoreMalformedStoredValues.storedField(name(), malformedDataForSyntheticSource)); } + context.addIgnoredField(fieldType().name()); } context.path().remove(); @@ -491,76 +514,85 @@ protected SyntheticSourceMode syntheticSourceMode() { @Override public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { - if (ignoreMalformed.value()) { - throw new IllegalArgumentException( - "field [" + name() + "] of type [histogram] doesn't support synthetic source because it ignores malformed histograms" - ); - } if (copyTo.copyToFields().isEmpty() != true) { throw new IllegalArgumentException( "field [" + name() + "] of type [histogram] doesn't support synthetic source because it declares copy_to" ); } - return new SourceLoader.SyntheticFieldLoader() { - private final InternalHistogramValue value = new InternalHistogramValue(); - private BytesRef binaryValue; - @Override - public Stream> storedFieldLoaders() { - return Stream.of(); - } + return new CompositeSyntheticFieldLoader( + simpleName(), + name(), + new HistogramSyntheticFieldLoader(), + new CompositeSyntheticFieldLoader.MalformedValuesLayer(name()) + ); + } - @Override - public DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) throws IOException { - BinaryDocValues docValues = leafReader.getBinaryDocValues(fieldType().name()); - if (docValues == null) { - // No values in this leaf - binaryValue = null; - return null; - } - return docId -> { - if (docValues.advanceExact(docId)) { - binaryValue = docValues.binaryValue(); - return true; - } - binaryValue = null; - return false; - }; - } + private class HistogramSyntheticFieldLoader implements CompositeSyntheticFieldLoader.SyntheticFieldLoaderLayer { + private final InternalHistogramValue value = new InternalHistogramValue(); + private BytesRef binaryValue; - @Override - public boolean hasValue() { - return binaryValue != null; - } + @Override + public Stream> storedFieldLoaders() { + return Stream.of(); + } - @Override - public void write(XContentBuilder b) throws IOException { - if (binaryValue == null) { - return; + @Override + public SourceLoader.SyntheticFieldLoader.DocValuesLoader docValuesLoader(LeafReader leafReader, int[] docIdsInLeaf) + throws IOException { + BinaryDocValues docValues = leafReader.getBinaryDocValues(fieldType().name()); + if (docValues == null) { + // No values in this leaf + binaryValue = null; + return null; + } + return docId -> { + if (docValues.advanceExact(docId)) { + binaryValue = docValues.binaryValue(); + return true; } - b.startObject(simpleName()); + binaryValue = null; + return false; + }; + } - value.reset(binaryValue); - b.startArray("values"); - while (value.next()) { - b.value(value.value()); - } - b.endArray(); + @Override + public boolean hasValue() { + return binaryValue != null; + } - value.reset(binaryValue); - b.startArray("counts"); - while (value.next()) { - b.value(value.count()); - } - b.endArray(); + @Override + public void write(XContentBuilder b) throws IOException { + if (binaryValue == null) { + return; + } + b.startObject(); - b.endObject(); + value.reset(binaryValue); + b.startArray("values"); + while (value.next()) { + b.value(value.value()); } + b.endArray(); - @Override - public String fieldName() { - return name(); + value.reset(binaryValue); + b.startArray("counts"); + while (value.next()) { + b.value(value.count()); } - }; - } + b.endArray(); + + b.endObject(); + } + + @Override + public String fieldName() { + return name(); + } + + @Override + public long valueCount() { + return binaryValue != null ? 1 : 0; + } + }; } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 5e2bdaf2d465e..6fcbf20b8657f 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.analytics.mapper; +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.MappedFieldType; @@ -15,6 +17,7 @@ import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.analytics.AnalyticsPlugin; import org.junit.AssumptionViolatedException; @@ -26,7 +29,6 @@ import java.util.Map; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.matchesPattern; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -121,9 +123,44 @@ protected boolean supportsIgnoreMalformed() { @Override protected List exampleMalformedValues() { + var randomString = randomAlphaOfLengthBetween(1, 10); + var randomLong = randomLong(); + var randomDouble = randomDouble(); + var randomBoolean = randomBoolean(); + return List.of( + exampleMalformedValue(b -> b.value(randomString)).errorMatches( + "Failed to parse object: expecting token of type [START_OBJECT]" + ), + exampleMalformedValue(b -> b.value(randomLong)).errorMatches("Failed to parse object: expecting token of type [START_OBJECT]"), + exampleMalformedValue(b -> b.value(randomDouble)).errorMatches( + "Failed to parse object: expecting token of type [START_OBJECT]" + ), + exampleMalformedValue(b -> b.value(randomBoolean)).errorMatches( + "Failed to parse object: expecting token of type [START_OBJECT]" + ), + exampleMalformedValue(b -> b.startObject().endObject()).errorMatches("expected field called [values]"), exampleMalformedValue(b -> b.startObject().startArray("values").value(2).value(2).endArray().endObject()).errorMatches( "expected field called [counts]" + ), + exampleMalformedValue(b -> b.startObject().startArray("counts").value(2).value(2).endArray().endObject()).errorMatches( + "expected field called [values]" + ), + // Make sure that entire sub-object is preserved in synthetic source + exampleMalformedValue( + b -> b.startObject() + .startArray("values") + .value(2) + .endArray() + .field("somefield", randomString) + .array("somearray", randomLong, randomLong) + .startObject("someobject") + .field("nestedfield", randomDouble) + .endObject() + .endObject() + ).errorMatches("unknown parameter [somefield]"), + exampleMalformedValue(b -> b.startArray().value(randomLong).value(randomLong).endArray()).errorMatches( + "expecting token of type [START_OBJECT] but found [VALUE_NUMBER]" ) ); } @@ -336,13 +373,44 @@ protected IngestScriptSupport ingestScriptSupport() { throw new AssumptionViolatedException("not supported"); } + public void testArrayValueSyntheticSource() throws Exception { + DocumentMapper mapper = createDocumentMapper( + syntheticSourceFieldMapping(b -> b.field("type", "histogram").field("ignore_malformed", "true")) + ); + + var randomString = randomAlphaOfLength(10); + CheckedConsumer arrayValue = b -> { + b.startArray("field"); + { + b.startObject().field("counts", new int[] { 1, 2, 3 }).field("values", new double[] { 1, 2, 3 }).endObject(); + b.startObject().field("counts", new int[] { 4, 5, 6 }).field("values", new double[] { 4, 5, 6 }).endObject(); + b.value(randomString); + } + b.endArray(); + }; + + var expected = JsonXContent.contentBuilder().startObject(); + // First value comes from synthetic field loader and so is formatted in a specific format (e.g. values always come first). + // Other values are stored as is as part of ignore_malformed logic for synthetic source. + { + expected.startArray("field"); + expected.startObject().field("values", new double[] { 1, 2, 3 }).field("counts", new int[] { 1, 2, 3 }).endObject(); + expected.startObject().field("counts", new int[] { 4, 5, 6 }).field("values", new double[] { 4, 5, 6 }).endObject(); + expected.value(randomString); + expected.endArray(); + } + expected.endObject(); + + var syntheticSource = syntheticSource(mapper, arrayValue); + assertEquals(Strings.toString(expected), syntheticSource); + } + @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { - assumeFalse("synthetic _source support for histogram doesn't support ignore_malformed", ignoreMalformed); - return new HistogramFieldSyntheticSourceSupport(); + return new HistogramFieldSyntheticSourceSupport(ignoreMalformed); } - private static class HistogramFieldSyntheticSourceSupport implements SyntheticSourceSupport { + private record HistogramFieldSyntheticSourceSupport(boolean ignoreMalformed) implements SyntheticSourceSupport { @Override public SyntheticSourceExample example(int maxVals) { if (randomBoolean()) { @@ -371,21 +439,14 @@ private int randomCount() { private void mapping(XContentBuilder b) throws IOException { b.field("type", "histogram"); + if (ignoreMalformed) { + b.field("ignore_malformed", true); + } } @Override public List invalidExample() throws IOException { - return List.of( - new SyntheticSourceInvalidExample( - matchesPattern( - "field \\[field] of type \\[histogram] doesn't support synthetic source because it ignores malformed histograms" - ), - b -> { - b.field("type", "histogram"); - b.field("ignore_malformed", true); - } - ) - ); + return List.of(); } } } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index ed7587556bd28..560c98fbd210b 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -334,6 +334,9 @@ static DataStream updateLocalDataStream( // (and potentially even break things). remoteDataStream.getBackingIndices().copy().setIndices(List.of(backingIndexToFollow)).setRolloverOnWrite(false).build() ) + // Replicated data streams should not have the failure store marked for lazy rollover (which they do by default for lazy + // failure store creation). + .setFailureIndices(remoteDataStream.getFailureIndices().copy().setRolloverOnWrite(false).build()) .setReplicated(true) .build(); } else { diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java index 7c9b1b5efbde2..fbddfc7683d2f 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/index/engine/FollowingEngineTests.java @@ -252,7 +252,8 @@ public void onFailedEngine(String reason, Exception e) { null, System::nanoTime, null, - true + true, + null ); } diff --git a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java index 083850e80dd47..4ff7a149bc8f0 100644 --- a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java +++ b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java @@ -74,6 +74,9 @@ public void testDSXpackUsage() throws Exception { indexRequest = new Request("POST", "/fs/_doc"); indexRequest.setJsonEntity("{\"@timestamp\": \"2020-01-01\"}"); client().performRequest(indexRequest); + // Initialize the failure store + rollover = new Request("POST", "/fs/_rollover?target_failure_store=true"); + client().performRequest(rollover); dataStreams = (Map) getLocation("/_xpack/usage").get("data_streams"); assertNotNull(dataStreams); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java index 251af69e1aaf5..0a13aab82aced 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/index/engine/frozen/FrozenEngine.java @@ -25,9 +25,11 @@ import org.elasticsearch.index.engine.EngineException; import org.elasticsearch.index.engine.ReadOnlyEngine; import org.elasticsearch.index.engine.SegmentsStats; +import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.seqno.SeqNoStats; import org.elasticsearch.index.shard.DenseVectorStats; import org.elasticsearch.index.shard.DocsStats; +import org.elasticsearch.index.shard.SparseVectorStats; import org.elasticsearch.index.store.Store; import org.elasticsearch.index.translog.TranslogStats; import org.elasticsearch.indices.ESCacheHelper; @@ -62,6 +64,7 @@ public final class FrozenEngine extends ReadOnlyEngine { private final SegmentsStats segmentsStats; private final DocsStats docsStats; private final DenseVectorStats denseVectorStats; + private final SparseVectorStats sparseVectorStats; private volatile ElasticsearchDirectoryReader lastOpenedReader; private final ElasticsearchDirectoryReader canMatchReader; private final Object cacheIdentity = new Object(); @@ -93,6 +96,7 @@ public FrozenEngine( } this.docsStats = docsStats(reader); this.denseVectorStats = denseVectorStats(reader); + this.sparseVectorStats = sparseVectorStats(reader, null); canMatchReader = ElasticsearchDirectoryReader.wrap( new RewriteCachingDirectoryReader(directory, reader.leaves(), null), config.getShardId() @@ -334,6 +338,11 @@ public DenseVectorStats denseVectorStats() { return denseVectorStats; } + @Override + public SparseVectorStats sparseVectorStats(MappingLookup mappingLookup) { + return sparseVectorStats; + } + synchronized boolean isReaderOpen() { return lastOpenedReader != null; } // this is mainly for tests diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 120ef76561a61..4f8a18e28aea1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -58,7 +58,10 @@ public class XPackLicenseState { messages.put(XPackField.DEPRECATION, new String[] { "Deprecation APIs are disabled" }); messages.put(XPackField.UPGRADE, new String[] { "Upgrade API is disabled" }); messages.put(XPackField.SQL, new String[] { "SQL support is disabled" }); - messages.put(XPackField.ENTERPRISE_SEARCH, new String[] { "Search Applications and behavioral analytics will be disabled" }); + messages.put( + XPackField.ENTERPRISE_SEARCH, + new String[] { "Search Applications, query rules and behavioral analytics will be disabled" } + ); messages.put( XPackField.ROLLUP, new String[] { @@ -222,11 +225,16 @@ private static String[] enterpriseSearchAcknowledgementMessages(OperationMode cu case STANDARD: case GOLD: switch (currentMode) { - case TRIAL: case PLATINUM: + return new String[] { + "Search Applications and behavioral analytics will be disabled.", + "Elastic Web crawler will be disabled.", + "Connector clients require at least a platinum license." }; + case TRIAL: case ENTERPRISE: return new String[] { "Search Applications and behavioral analytics will be disabled.", + "Query rules will be disabled.", "Elastic Web crawler will be disabled.", "Connector clients require at least a platinum license." }; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/time/RemainingTime.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/time/RemainingTime.java new file mode 100644 index 0000000000000..33a3f2424c90c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/common/time/RemainingTime.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.common.time; + +import org.elasticsearch.core.TimeValue; + +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.function.Supplier; + +public interface RemainingTime extends Supplier { + /** + * Create a {@link Supplier} that returns a decreasing {@link TimeValue} on each invocation, representing the amount of time until + * the call times out. The timer starts when this method is called and counts down from remainingTime to 0. + * currentTime should return the most up-to-date system time, for example Instant.now() or Clock.instant(). + */ + static RemainingTime from(Supplier currentTime, TimeValue remainingTime) { + var timeout = currentTime.get().plus(remainingTime.duration(), remainingTime.timeUnit().toChronoUnit()); + var maxRemainingTime = remainingTime.nanos(); + return () -> { + var remainingNanos = ChronoUnit.NANOS.between(currentTime.get(), timeout); + return TimeValue.timeValueNanos(Math.max(0, Math.min(remainingNanos, maxRemainingTime))); + }; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java index c20d97b7125a0..b89b73f58c9b2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/NodesDataTiersUsageTransportAction.java @@ -170,11 +170,6 @@ public NodesRequest() { public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { return new CancellableTask(id, type, action, "", parentTaskId, headers); } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } } public static class NodeRequest extends TransportRequest { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsAction.java index 00dcd56424016..ef7af21e5e133 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/action/GetInferenceDiagnosticsAction.java @@ -10,7 +10,6 @@ import org.apache.http.pool.PoolStats; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.action.support.nodes.BaseNodesResponse; @@ -55,11 +54,6 @@ public int hashCode() { // The class doesn't have any members at the moment so return the same hash code return Objects.hash(NAME); } - - @Override - public void writeTo(StreamOutput out) { - TransportAction.localOnly(); - } } public static class NodeRequest extends TransportRequest { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java index 2093b687a2ab9..f1265873ad6dd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedSparseEmbeddingResults.java @@ -15,7 +15,7 @@ import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContent; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import java.io.IOException; @@ -33,7 +33,7 @@ public class InferenceChunkedSparseEmbeddingResults implements ChunkedInferenceS public static final String NAME = "chunked_sparse_embedding_results"; public static final String FIELD_NAME = "sparse_embedding_chunk"; - public static InferenceChunkedSparseEmbeddingResults ofMlResult(InferenceChunkedTextExpansionResults mlInferenceResults) { + public static InferenceChunkedSparseEmbeddingResults ofMlResult(MlChunkedTextExpansionResults mlInferenceResults) { return new InferenceChunkedSparseEmbeddingResults(mlInferenceResults.getChunks()); } @@ -59,29 +59,27 @@ private static InferenceChunkedSparseEmbeddingResults ofSingle(String input, Spa .map(weightedToken -> new WeightedToken(weightedToken.token(), weightedToken.weight())) .toList(); - return new InferenceChunkedSparseEmbeddingResults( - List.of(new InferenceChunkedTextExpansionResults.ChunkedResult(input, weightedTokens)) - ); + return new InferenceChunkedSparseEmbeddingResults(List.of(new MlChunkedTextExpansionResults.ChunkedResult(input, weightedTokens))); } - private final List chunkedResults; + private final List chunkedResults; - public InferenceChunkedSparseEmbeddingResults(List chunks) { + public InferenceChunkedSparseEmbeddingResults(List chunks) { this.chunkedResults = chunks; } public InferenceChunkedSparseEmbeddingResults(StreamInput in) throws IOException { - this.chunkedResults = in.readCollectionAsList(InferenceChunkedTextExpansionResults.ChunkedResult::new); + this.chunkedResults = in.readCollectionAsList(MlChunkedTextExpansionResults.ChunkedResult::new); } - public List getChunkedResults() { + public List getChunkedResults() { return chunkedResults; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startArray(FIELD_NAME); - for (InferenceChunkedTextExpansionResults.ChunkedResult chunk : chunkedResults) { + for (MlChunkedTextExpansionResults.ChunkedResult chunk : chunkedResults) { chunk.toXContent(builder, params); } builder.endArray(); @@ -112,7 +110,7 @@ public List transformToLegacyFormat() { public Map asMap() { return Map.of( FIELD_NAME, - chunkedResults.stream().map(InferenceChunkedTextExpansionResults.ChunkedResult::asMap).collect(Collectors.toList()) + chunkedResults.stream().map(MlChunkedTextExpansionResults.ChunkedResult::asMap).collect(Collectors.toList()) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java index a2bc072064ea1..b78bce8c5c2cd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingByteResults.java @@ -162,6 +162,7 @@ public int hashCode() { } } + @Override public Iterator chunksAsMatchedTextAndByteReference(XContent xcontent) { return chunks.stream().map(chunk -> new Chunk(chunk.matchedText(), toBytesReference(xcontent, chunk.embedding()))).iterator(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java index 9b625f9b1712a..9fead334dcbc0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/inference/results/InferenceChunkedTextEmbeddingFloatResults.java @@ -178,6 +178,7 @@ public int hashCode() { } } + @Override public Iterator chunksAsMatchedTextAndByteReference(XContent xcontent) { return chunks.stream().map(chunk -> new Chunk(chunk.matchedText(), toBytesReference(xcontent, chunk.embedding()))).iterator(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java index b3cf9f16c3c82..ca9b86a90f875 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartTrainedModelDeploymentAction.java @@ -67,6 +67,11 @@ public class StartTrainedModelDeploymentAction extends ActionType { + private final DiscoveryNode[] concreteNodes; + public Request(DiscoveryNode... concreteNodes) { super(concreteNodes); - } - - @UpdateForV9 // this constructor is unused in v9 - public Request(StreamInput in) throws IOException { - super(in); - } - - @UpdateForV9 // this method can just call localOnly() in v9 - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); + this.concreteNodes = concreteNodes; } @Override public int hashCode() { - return Arrays.hashCode(concreteNodes()); + return Arrays.hashCode(concreteNodes); } @Override @@ -65,7 +56,7 @@ public boolean equals(Object obj) { return false; } Request other = (Request) obj; - return Arrays.deepEquals(concreteNodes(), other.concreteNodes()); + return Arrays.deepEquals(concreteNodes, other.concreteNodes); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java index 354e898a514d7..65e30072d9870 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/MlInferenceNamedXContentProvider.java @@ -23,8 +23,8 @@ import org.elasticsearch.xpack.core.ml.inference.results.ClassificationInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.core.ml.inference.results.FillMaskResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.MlTextEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.NerResults; import org.elasticsearch.xpack.core.ml.inference.results.NlpClassificationInferenceResults; @@ -684,11 +684,7 @@ public List getNamedWriteables() { ) ); namedWriteables.add( - new NamedWriteableRegistry.Entry( - InferenceResults.class, - InferenceChunkedTextExpansionResults.NAME, - InferenceChunkedTextExpansionResults::new - ) + new NamedWriteableRegistry.Entry(InferenceResults.class, MlChunkedTextExpansionResults.NAME, MlChunkedTextExpansionResults::new) ); // Inference Configs diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceChunkedTextExpansionResults.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/MlChunkedTextExpansionResults.java similarity index 91% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceChunkedTextExpansionResults.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/MlChunkedTextExpansionResults.java index 3c719262fbfc6..bdaa5d792e9ca 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceChunkedTextExpansionResults.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/results/MlChunkedTextExpansionResults.java @@ -21,7 +21,7 @@ import java.util.Objects; import java.util.stream.Collectors; -public class InferenceChunkedTextExpansionResults extends ChunkedNlpInferenceResults { +public class MlChunkedTextExpansionResults extends ChunkedNlpInferenceResults { public static final String NAME = "chunked_text_expansion_result"; public record ChunkedResult(String matchedText, List weightedTokens) implements Writeable, ToXContentObject { @@ -60,13 +60,13 @@ public Map asMap() { private final String resultsField; private final List chunks; - public InferenceChunkedTextExpansionResults(String resultField, List chunks, boolean isTruncated) { + public MlChunkedTextExpansionResults(String resultField, List chunks, boolean isTruncated) { super(isTruncated); this.resultsField = resultField; this.chunks = chunks; } - public InferenceChunkedTextExpansionResults(StreamInput in) throws IOException { + public MlChunkedTextExpansionResults(StreamInput in) throws IOException { super(in); this.resultsField = in.readString(); this.chunks = in.readCollectionAsList(ChunkedResult::new); @@ -104,7 +104,7 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; if (super.equals(o) == false) return false; - InferenceChunkedTextExpansionResults that = (InferenceChunkedTextExpansionResults) o; + MlChunkedTextExpansionResults that = (MlChunkedTextExpansionResults) o; return Objects.equals(resultsField, that.resultsField) && Objects.equals(chunks, that.chunks); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java index 3812c012e2a3d..16eceb1e89a95 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStats.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.core.ml.job.process.autodetect.state; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -48,6 +49,7 @@ public class ModelSizeStats implements ToXContentObject, Writeable { public static final ParseField BUCKET_ALLOCATION_FAILURES_COUNT_FIELD = new ParseField("bucket_allocation_failures_count"); public static final ParseField MEMORY_STATUS_FIELD = new ParseField("memory_status"); public static final ParseField ASSIGNMENT_MEMORY_BASIS_FIELD = new ParseField("assignment_memory_basis"); + public static final ParseField OUTPUT_MEMORY_ALLOCATOR_BYTES_FIELD = new ParseField("output_memory_allocator_bytes"); public static final ParseField CATEGORIZED_DOC_COUNT_FIELD = new ParseField("categorized_doc_count"); public static final ParseField TOTAL_CATEGORY_COUNT_FIELD = new ParseField("total_category_count"); public static final ParseField FREQUENT_CATEGORY_COUNT_FIELD = new ParseField("frequent_category_count"); @@ -85,6 +87,7 @@ private static ConstructingObjectParser createParser(boolean igno ASSIGNMENT_MEMORY_BASIS_FIELD, ValueType.STRING ); + parser.declareLong(Builder::setOutputMemoryAllocatorBytes, OUTPUT_MEMORY_ALLOCATOR_BYTES_FIELD); parser.declareLong(Builder::setCategorizedDocCount, CATEGORIZED_DOC_COUNT_FIELD); parser.declareLong(Builder::setTotalCategoryCount, TOTAL_CATEGORY_COUNT_FIELD); parser.declareLong(Builder::setFrequentCategoryCount, FREQUENT_CATEGORY_COUNT_FIELD); @@ -188,6 +191,7 @@ public String toString() { private final long bucketAllocationFailuresCount; private final MemoryStatus memoryStatus; private final AssignmentMemoryBasis assignmentMemoryBasis; + private final Long outputMemoryAllocatorBytes; private final long categorizedDocCount; private final long totalCategoryCount; private final long frequentCategoryCount; @@ -210,6 +214,7 @@ private ModelSizeStats( long bucketAllocationFailuresCount, MemoryStatus memoryStatus, AssignmentMemoryBasis assignmentMemoryBasis, + Long outputMemoryAllocatorBytes, long categorizedDocCount, long totalCategoryCount, long frequentCategoryCount, @@ -231,6 +236,7 @@ private ModelSizeStats( this.bucketAllocationFailuresCount = bucketAllocationFailuresCount; this.memoryStatus = memoryStatus; this.assignmentMemoryBasis = assignmentMemoryBasis; + this.outputMemoryAllocatorBytes = outputMemoryAllocatorBytes; this.categorizedDocCount = categorizedDocCount; this.totalCategoryCount = totalCategoryCount; this.frequentCategoryCount = frequentCategoryCount; @@ -258,6 +264,11 @@ public ModelSizeStats(StreamInput in) throws IOException { } else { assignmentMemoryBasis = null; } + if (in.getTransportVersion().onOrAfter(TransportVersions.ML_AD_OUTPUT_MEMORY_ALLOCATOR_FIELD)) { + outputMemoryAllocatorBytes = in.readOptionalVLong(); + } else { + outputMemoryAllocatorBytes = null; + } categorizedDocCount = in.readVLong(); totalCategoryCount = in.readVLong(); frequentCategoryCount = in.readVLong(); @@ -295,6 +306,9 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } + if (out.getTransportVersion().onOrAfter(TransportVersions.ML_AD_OUTPUT_MEMORY_ALLOCATOR_FIELD)) { + out.writeOptionalVLong(outputMemoryAllocatorBytes); + } out.writeVLong(categorizedDocCount); out.writeVLong(totalCategoryCount); out.writeVLong(frequentCategoryCount); @@ -339,6 +353,9 @@ public XContentBuilder doXContentBody(XContentBuilder builder) throws IOExceptio if (assignmentMemoryBasis != null) { builder.field(ASSIGNMENT_MEMORY_BASIS_FIELD.getPreferredName(), assignmentMemoryBasis); } + if (outputMemoryAllocatorBytes != null) { + builder.field(OUTPUT_MEMORY_ALLOCATOR_BYTES_FIELD.getPreferredName(), outputMemoryAllocatorBytes); + } builder.field(CATEGORIZED_DOC_COUNT_FIELD.getPreferredName(), categorizedDocCount); builder.field(TOTAL_CATEGORY_COUNT_FIELD.getPreferredName(), totalCategoryCount); builder.field(FREQUENT_CATEGORY_COUNT_FIELD.getPreferredName(), frequentCategoryCount); @@ -399,6 +416,10 @@ public AssignmentMemoryBasis getAssignmentMemoryBasis() { return assignmentMemoryBasis; } + public Long getOutputMemmoryAllocatorBytes() { + return outputMemoryAllocatorBytes; + } + public long getCategorizedDocCount() { return categorizedDocCount; } @@ -458,6 +479,7 @@ public int hashCode() { bucketAllocationFailuresCount, memoryStatus, assignmentMemoryBasis, + outputMemoryAllocatorBytes, categorizedDocCount, totalCategoryCount, frequentCategoryCount, @@ -495,6 +517,7 @@ public boolean equals(Object other) { && this.bucketAllocationFailuresCount == that.bucketAllocationFailuresCount && Objects.equals(this.memoryStatus, that.memoryStatus) && Objects.equals(this.assignmentMemoryBasis, that.assignmentMemoryBasis) + && Objects.equals(this.outputMemoryAllocatorBytes, that.outputMemoryAllocatorBytes) && Objects.equals(this.categorizedDocCount, that.categorizedDocCount) && Objects.equals(this.totalCategoryCount, that.totalCategoryCount) && Objects.equals(this.frequentCategoryCount, that.frequentCategoryCount) @@ -520,6 +543,7 @@ public static class Builder { private long bucketAllocationFailuresCount; private MemoryStatus memoryStatus; private AssignmentMemoryBasis assignmentMemoryBasis; + private Long outputMemoryAllocatorBytes; private long categorizedDocCount; private long totalCategoryCount; private long frequentCategoryCount; @@ -549,6 +573,7 @@ public Builder(ModelSizeStats modelSizeStats) { this.bucketAllocationFailuresCount = modelSizeStats.bucketAllocationFailuresCount; this.memoryStatus = modelSizeStats.memoryStatus; this.assignmentMemoryBasis = modelSizeStats.assignmentMemoryBasis; + this.outputMemoryAllocatorBytes = modelSizeStats.outputMemoryAllocatorBytes; this.categorizedDocCount = modelSizeStats.categorizedDocCount; this.totalCategoryCount = modelSizeStats.totalCategoryCount; this.frequentCategoryCount = modelSizeStats.frequentCategoryCount; @@ -611,6 +636,11 @@ public Builder setAssignmentMemoryBasis(AssignmentMemoryBasis assignmentMemoryBa return this; } + public Builder setOutputMemoryAllocatorBytes(long outputMemoryAllocatorBytes) { + this.outputMemoryAllocatorBytes = outputMemoryAllocatorBytes; + return this; + } + public Builder setCategorizedDocCount(long categorizedDocCount) { this.categorizedDocCount = categorizedDocCount; return this; @@ -670,6 +700,7 @@ public ModelSizeStats build() { bucketAllocationFailuresCount, memoryStatus, assignmentMemoryBasis, + outputMemoryAllocatorBytes, categorizedDocCount, totalCategoryCount, frequentCategoryCount, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlPlatformArchitecturesUtil.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlPlatformArchitecturesUtil.java index c0f00cdada28f..e7358cafd3fc9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlPlatformArchitecturesUtil.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/MlPlatformArchitecturesUtil.java @@ -54,7 +54,7 @@ static ActionListener getArchitecturesSetFromNodesInfoRespons } static NodesInfoRequestBuilder getNodesInfoBuilderWithMlNodeArchitectureInfo(Client client) { - return client.admin().cluster().prepareNodesInfo().clear().setNodesIds("ml:true").setOs(true).setPlugins(true); + return client.admin().cluster().prepareNodesInfo("ml:true").clear().setOs(true).setPlugins(true); } private static Set getArchitecturesSetFromNodesInfoResponse(NodesInfoResponse nodesInfoResponse) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ClearSecurityCacheRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ClearSecurityCacheRequest.java index 78ec5ddaa1ad5..e97670f20bfbc 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ClearSecurityCacheRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/ClearSecurityCacheRequest.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.core.security.action; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -42,11 +41,6 @@ public String[] keys() { return keys; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - public static class Node extends TransportRequest { private String cacheName; private String[] keys; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ClearPrivilegesCacheRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ClearPrivilegesCacheRequest.java index 540cc4b3c70cd..6857f43bda25e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ClearPrivilegesCacheRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/privilege/ClearPrivilegesCacheRequest.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.core.security.action.privilege; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -42,11 +41,6 @@ public boolean clearRolesCache() { return clearRolesCache; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - public static class Node extends TransportRequest { private String[] applicationNames; private boolean clearRolesCache; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheRequest.java index ceee6cea8481a..fe381fc09b74a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/realm/ClearRealmCacheRequest.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.security.action.realm; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -73,11 +72,6 @@ public ClearRealmCacheRequest usernames(String... usernames) { return this; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - public static class Node extends TransportRequest { private String[] realms; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheRequest.java index 0d06382a891da..74d3a2ac85c78 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/role/ClearRolesCacheRequest.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.security.action.role; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -42,11 +41,6 @@ public String[] names() { return names; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - public static class Node extends TransportRequest { private String[] names; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountCredentialsNodesRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountCredentialsNodesRequest.java index a2ebb338c15f0..9431ea14097a2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountCredentialsNodesRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/service/GetServiceAccountCredentialsNodesRequest.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.core.security.action.service; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -30,11 +29,6 @@ public GetServiceAccountCredentialsNodesRequest(String namespace, String service this.serviceName = serviceName; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - public static class Node extends TransportRequest { private final String namespace; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestBuilderFactory.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestBuilderFactory.java index e610e40333da8..04dd9e31e519c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestBuilderFactory.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/user/HasPrivilegesRequestBuilderFactory.java @@ -10,13 +10,12 @@ import org.elasticsearch.client.internal.Client; public interface HasPrivilegesRequestBuilderFactory { - HasPrivilegesRequestBuilder create(Client client, boolean restrictRequest); + HasPrivilegesRequestBuilder create(Client client); class Default implements HasPrivilegesRequestBuilderFactory { @Override - public HasPrivilegesRequestBuilder create(Client client, boolean restrictRequest) { - assert false == restrictRequest; + public HasPrivilegesRequestBuilder create(Client client) { return new HasPrivilegesRequestBuilder(client); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/action/SpatialStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/action/SpatialStatsAction.java index 117d613a20cd9..80698dcbe022c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/action/SpatialStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/spatial/action/SpatialStatsAction.java @@ -8,7 +8,6 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.action.FailedNodeException; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.action.support.nodes.BaseNodesResponse; @@ -75,11 +74,6 @@ public boolean equals(Object obj) { } return true; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } } public static class NodeRequest extends TransportRequest { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformNodeStatsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformNodeStatsAction.java index 6cadefbe206f0..2ae4593ed3baa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformNodeStatsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/GetTransformNodeStatsAction.java @@ -42,15 +42,9 @@ private GetTransformNodeStatsAction() { } public static class NodesStatsRequest extends BaseNodesRequest { - public NodesStatsRequest() { super(Strings.EMPTY_ARRAY); } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } } public static class NodesStatsResponse extends BaseNodesResponse implements ToXContentObject { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsRequest.java index ac55db16802d2..2162ec5c38cec 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/transport/actions/stats/WatcherStatsRequest.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.watcher.transport.actions.stats; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -51,11 +50,6 @@ public void includeStats(boolean includeStats) { this.includeStats = includeStats; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - @Override public String toString() { return "watcher_stats"; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/time/RemainingTimeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/time/RemainingTimeTests.java new file mode 100644 index 0000000000000..3a948608f6ae3 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/common/time/RemainingTimeTests.java @@ -0,0 +1,40 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.common.time; + +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.time.Instant; +import java.util.Arrays; +import java.util.function.Supplier; +import java.util.stream.Stream; + +public class RemainingTimeTests extends ESTestCase { + public void testRemainingTime() { + var remainingTime = RemainingTime.from(times(Instant.now(), Instant.now().plusSeconds(60)), TimeValue.timeValueSeconds(30)); + assertThat(remainingTime.get(), Matchers.greaterThan(TimeValue.ZERO)); + assertThat(remainingTime.get(), Matchers.equalTo(TimeValue.ZERO)); + } + + public void testRemainingTimeMaxValue() { + var remainingTime = RemainingTime.from( + times(Instant.now().minusSeconds(60), Instant.now().plusSeconds(60)), + TimeValue.timeValueSeconds(30) + ); + assertThat(remainingTime.get(), Matchers.equalTo(TimeValue.timeValueSeconds(30))); + assertThat(remainingTime.get(), Matchers.equalTo(TimeValue.ZERO)); + } + + // always add the first value, which is read when RemainingTime.from is called, then add the test values + private Supplier times(Instant... instants) { + var startTime = Stream.of(Instant.now()); + return Stream.concat(startTime, Arrays.stream(instants)).iterator()::next; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoRequestTests.java deleted file mode 100644 index 8620b8d77755c..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/TrainedModelCacheInfoRequestTests.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.core.ml.action; - -import org.elasticsearch.cluster.node.DiscoveryNode; -import org.elasticsearch.cluster.node.DiscoveryNodeUtils; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -import java.net.InetAddress; - -public class TrainedModelCacheInfoRequestTests extends AbstractWireSerializingTestCase { - - @Override - protected Writeable.Reader instanceReader() { - return TrainedModelCacheInfoAction.Request::new; - } - - @Override - protected TrainedModelCacheInfoAction.Request createTestInstance() { - int numNodes = randomIntBetween(1, 20); - DiscoveryNode[] nodes = new DiscoveryNode[numNodes]; - for (int i = 0; i < numNodes; ++i) { - nodes[i] = DiscoveryNodeUtils.create(randomAlphaOfLength(20), new TransportAddress(InetAddress.getLoopbackAddress(), 9200 + i)); - } - return new TrainedModelCacheInfoAction.Request(nodes); - } - - @Override - protected TrainedModelCacheInfoAction.Request mutateInstance(TrainedModelCacheInfoAction.Request instance) { - return null;// TODO implement https://github.com/elastic/elasticsearch/issues/25929 - } -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceChunkedTextExpansionResultsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceChunkedTextExpansionResultsTests.java index f5db7a2863e0c..9699b13acf3b9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceChunkedTextExpansionResultsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/inference/results/InferenceChunkedTextExpansionResultsTests.java @@ -16,10 +16,10 @@ import static org.elasticsearch.xpack.core.ml.inference.trainedmodel.InferenceConfig.DEFAULT_RESULTS_FIELD; -public class InferenceChunkedTextExpansionResultsTests extends AbstractWireSerializingTestCase { +public class InferenceChunkedTextExpansionResultsTests extends AbstractWireSerializingTestCase { - public static InferenceChunkedTextExpansionResults createRandomResults() { - var chunks = new ArrayList(); + public static MlChunkedTextExpansionResults createRandomResults() { + var chunks = new ArrayList(); int numChunks = randomIntBetween(1, 5); for (int i = 0; i < numChunks; i++) { @@ -28,24 +28,24 @@ public static InferenceChunkedTextExpansionResults createRandomResults() { for (int j = 0; j < numTokens; j++) { tokenWeights.add(new WeightedToken(Integer.toString(j), (float) randomDoubleBetween(0.0, 5.0, false))); } - chunks.add(new InferenceChunkedTextExpansionResults.ChunkedResult(randomAlphaOfLength(6), tokenWeights)); + chunks.add(new MlChunkedTextExpansionResults.ChunkedResult(randomAlphaOfLength(6), tokenWeights)); } - return new InferenceChunkedTextExpansionResults(DEFAULT_RESULTS_FIELD, chunks, randomBoolean()); + return new MlChunkedTextExpansionResults(DEFAULT_RESULTS_FIELD, chunks, randomBoolean()); } @Override - protected Writeable.Reader instanceReader() { - return InferenceChunkedTextExpansionResults::new; + protected Writeable.Reader instanceReader() { + return MlChunkedTextExpansionResults::new; } @Override - protected InferenceChunkedTextExpansionResults createTestInstance() { + protected MlChunkedTextExpansionResults createTestInstance() { return createRandomResults(); } @Override - protected InferenceChunkedTextExpansionResults mutateInstance(InferenceChunkedTextExpansionResults instance) throws IOException { + protected MlChunkedTextExpansionResults mutateInstance(MlChunkedTextExpansionResults instance) throws IOException { return null; } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java index 2279164a7cbea..91e2971f369e3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/process/autodetect/state/ModelSizeStatsTests.java @@ -31,6 +31,7 @@ public void testDefaultConstructor() { assertEquals(0, stats.getBucketAllocationFailuresCount()); assertEquals(MemoryStatus.OK, stats.getMemoryStatus()); assertNull(stats.getAssignmentMemoryBasis()); + assertNull(stats.getOutputMemmoryAllocatorBytes()); assertEquals(0, stats.getCategorizedDocCount()); assertEquals(0, stats.getTotalCategoryCount()); assertEquals(0, stats.getFrequentCategoryCount()); diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json new file mode 100644 index 0000000000000..167efbd3ffaf5 --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@mappings-logsdb.json @@ -0,0 +1,31 @@ +{ + "template": { + "mappings": { + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "data_stream.type": { + "type": "constant_keyword", + "value": "logs" + }, + "data_stream.dataset": { + "type": "constant_keyword" + }, + "data_stream.namespace": { + "type": "constant_keyword" + } + } + } + }, + "_meta": { + "description": "default mappings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json new file mode 100644 index 0000000000000..b02866e867c4a --- /dev/null +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings-logsdb.json @@ -0,0 +1,26 @@ +{ + "template": { + "settings": { + "index": { + "mode": "logs", + "lifecycle": { + "name": "logs" + }, + "codec": "best_compression", + "mapping": { + "ignore_malformed": true, + "total_fields": { + "ignore_dynamic_beyond_limit": true + } + }, + "default_pipeline": "logs@default-pipeline" + } + } + }, + "_meta": { + "description": "default settings for the logs index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version}, + "deprecated": ${xpack.stack.template.deprecated} +} diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java index c867ef671811c..1d9fb86998b9b 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckAction.java @@ -21,6 +21,9 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.action.support.nodes.TransportNodesAction.sendLegacyNodesRequestHeader; +import static org.elasticsearch.action.support.nodes.TransportNodesAction.skipLegacyNodesRequestHeader; + /** * Runs deprecation checks on each node. Deprecation checks are performed locally so that filtered settings * can be accessed in the deprecation checks. @@ -40,17 +43,13 @@ public NodeRequest() {} public NodeRequest(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_REQUESTS)) { - new NodesDeprecationCheckRequest(in); - } + skipLegacyNodesRequestHeader(TransportVersions.DROP_UNUSED_NODES_REQUESTS, in); } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_REQUESTS)) { - new NodesDeprecationCheckRequest().writeTo(out); - } + sendLegacyNodesRequestHeader(TransportVersions.DROP_UNUSED_NODES_REQUESTS, out); } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequest.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequest.java index 2e5f77ee52778..ebe8f036c80a6 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequest.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequest.java @@ -8,31 +8,16 @@ package org.elasticsearch.xpack.deprecation; import org.elasticsearch.action.support.nodes.BaseNodesRequest; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.core.UpdateForV9; -import java.io.IOException; import java.util.Arrays; import java.util.Objects; public class NodesDeprecationCheckRequest extends BaseNodesRequest { - @UpdateForV9 // this constructor is unused in v9 - public NodesDeprecationCheckRequest(StreamInput in) throws IOException { - super(in); - } - public NodesDeprecationCheckRequest(String... nodesIds) { super(nodesIds); } - @UpdateForV9 // this method can just call localOnly() in v9 - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - } - @Override public int hashCode() { return Objects.hash((Object[]) this.nodesIds()); diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequestTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequestTests.java index a9e6e80df5040..24e2e9f76e125 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequestTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/NodesDeprecationCheckRequestTests.java @@ -7,25 +7,26 @@ package org.elasticsearch.xpack.deprecation; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.EqualsHashCodeTestUtils; -public class NodesDeprecationCheckRequestTests extends AbstractWireSerializingTestCase { +public class NodesDeprecationCheckRequestTests extends ESTestCase { - @Override - protected Writeable.Reader instanceReader() { - return NodesDeprecationCheckRequest::new; + public void testEqualsAndHashCode() { + EqualsHashCodeTestUtils.checkEqualsAndHashCode( + createTestInstance(), + i -> new NodesDeprecationCheckRequest(i.nodesIds()), + this::mutateInstance + ); } - @Override - protected NodesDeprecationCheckRequest mutateInstance(NodesDeprecationCheckRequest instance) { + private NodesDeprecationCheckRequest mutateInstance(NodesDeprecationCheckRequest instance) { int newSize = randomValueOtherThan(instance.nodesIds().length, () -> randomIntBetween(0, 10)); String[] newNodeIds = randomArray(newSize, newSize, String[]::new, () -> randomAlphaOfLengthBetween(5, 10)); return new NodesDeprecationCheckRequest(newNodeIds); } - @Override - protected NodesDeprecationCheckRequest createTestInstance() { + private NodesDeprecationCheckRequest createTestInstance() { return new NodesDeprecationCheckRequest(randomArray(0, 10, String[]::new, () -> randomAlphaOfLengthBetween(5, 10))); } } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java index f40f14059772e..d540cdb83361d 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichCoordinatorStatsAction.java @@ -49,11 +49,6 @@ public static class Request extends BaseNodesRequest { public Request() { super(new String[0]); } - - @Override - public void writeTo(StreamOutput out) { - org.elasticsearch.action.support.TransportAction.localOnly(); - } } public static class NodeRequest extends TransportRequest { @@ -136,9 +131,8 @@ public TransportAction( } @Override - protected void resolveRequest(Request request, ClusterState clusterState) { - DiscoveryNode[] ingestNodes = clusterState.getNodes().getIngestNodes().values().toArray(DiscoveryNode[]::new); - request.setConcreteNodes(ingestNodes); + protected DiscoveryNode[] resolveRequest(Request request, ClusterState clusterState) { + return clusterState.getNodes().getIngestNodes().values().toArray(DiscoveryNode[]::new); } @Override diff --git a/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/90_connector_sync_job_claim.yml b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/90_connector_sync_job_claim.yml new file mode 100644 index 0000000000000..39dd8eb05bc52 --- /dev/null +++ b/x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/connector/sync_job/90_connector_sync_job_claim.yml @@ -0,0 +1,113 @@ +setup: + - requires: + cluster_features: [ "gte_v8.15.0" ] + reason: Introduced in 8.15.0 + - do: + connector.put: + connector_id: test-connector + body: + index_name: search-test + name: my-connector + language: de + is_native: false + service_type: super-connector + + +--- +"Set only worker_hostname for a Connector Sync Job": + - do: + connector.sync_job_post: + body: + id: test-connector + job_type: full + trigger_method: on_demand + - set: { id: id } + + - do: + connector.sync_job_get: + connector_sync_job_id: $id + - match: { status: pending } + + - do: + connector.sync_job_claim: + connector_sync_job_id: $id + body: + worker_hostname: "host-name" + - match: { result: updated } + + - do: + connector.sync_job_get: + connector_sync_job_id: $id + + - match: { worker_hostname: "host-name" } + - match: { status: in_progress } + + - do: + connector.get: + connector_id: test-connector + + - match: { sync_cursor: null } + + +--- +"Set both worker_hostname and sync_cursor for a Connector Sync Job": + - do: + connector.sync_job_post: + body: + id: test-connector + job_type: full + trigger_method: on_demand + - set: { id: id } + + - do: + connector.sync_job_get: + connector_sync_job_id: $id + - match: { status: pending } + - do: + connector.sync_job_claim: + connector_sync_job_id: $id + body: + worker_hostname: "host-name" + sync_cursor: { cursor: "cursor" } + - match: { result: updated } + + - do: + connector.sync_job_get: + connector_sync_job_id: $id + + - match: { worker_hostname: "host-name" } + - match: { status: in_progress } + - match: { connector.sync_cursor: { cursor: "cursor" } } + + - do: + connector.get: + connector_id: test-connector + + - match: { sync_cursor: null } + +--- +"Fail to claim a Connector Sync Job - Connector Sync Job does not exist": + - do: + catch: "missing" + connector.sync_job_claim: + connector_sync_job_id: non-existing-connector-sync-job-id + body: + worker_hostname: "host-name" + +--- +"Fail to claim a Connector Sync Job - worker_hostname is missing": + - do: + catch: "bad_request" + connector.sync_job_claim: + connector_sync_job_id: test-connector + body: + sync_cursor: { cursor: "cursor" } + +--- +"Fail to claim a Connector Sync Job - worker_hostname is null": + - do: + catch: "bad_request" + connector.sync_job_claim: + connector_sync_job_id: test-connector + body: + worker_hostname: null diff --git a/x-pack/plugin/ent-search/src/main/java/module-info.java b/x-pack/plugin/ent-search/src/main/java/module-info.java index 5850b279f8b09..2acf0654dcdc3 100644 --- a/x-pack/plugin/ent-search/src/main/java/module-info.java +++ b/x-pack/plugin/ent-search/src/main/java/module-info.java @@ -37,6 +37,7 @@ exports org.elasticsearch.xpack.application.connector.action; exports org.elasticsearch.xpack.application.connector.syncjob; exports org.elasticsearch.xpack.application.connector.syncjob.action; + exports org.elasticsearch.xpack.application.utils; provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.application.EnterpriseSearchFeatures; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java index 078e242829e6b..bdd4cae3dda81 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearch.java @@ -123,12 +123,14 @@ import org.elasticsearch.xpack.application.connector.secrets.action.TransportPutConnectorSecretAction; import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction; +import org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.DeleteConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.GetConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.ListConnectorSyncJobsAction; import org.elasticsearch.xpack.application.connector.syncjob.action.PostConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestCancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestCheckInConnectorSyncJobAction; +import org.elasticsearch.xpack.application.connector.syncjob.action.RestClaimConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestDeleteConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestGetConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.RestListConnectorSyncJobsAction; @@ -137,6 +139,7 @@ import org.elasticsearch.xpack.application.connector.syncjob.action.RestUpdateConnectorSyncJobIngestionStatsAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportCancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportCheckInConnectorSyncJobAction; +import org.elasticsearch.xpack.application.connector.syncjob.action.TransportClaimConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportDeleteConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportGetConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.TransportListConnectorSyncJobsAction; @@ -310,7 +313,8 @@ protected XPackLicenseState getLicenseState() { new ActionHandler<>( UpdateConnectorSyncJobIngestionStatsAction.INSTANCE, TransportUpdateConnectorSyncJobIngestionStatsAction.class - ) + ), + new ActionHandler<>(ClaimConnectorSyncJobAction.INSTANCE, TransportClaimConnectorSyncJobAction.class) ) ); } @@ -408,7 +412,8 @@ public List getRestHandlers( new RestCheckInConnectorSyncJobAction(), new RestListConnectorSyncJobsAction(), new RestUpdateConnectorSyncJobErrorAction(), - new RestUpdateConnectorSyncJobIngestionStatsAction() + new RestUpdateConnectorSyncJobIngestionStatsAction(), + new RestClaimConnectorSyncJobAction() ) ); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchBaseRestHandler.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchBaseRestHandler.java index 0da68f206ee64..aa200f7ae9acb 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchBaseRestHandler.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchBaseRestHandler.java @@ -26,7 +26,7 @@ protected EnterpriseSearchBaseRestHandler(XPackLicenseState licenseState, Licens } protected final BaseRestHandler.RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { - if (LicenseUtils.supportedLicense(this.licenseState)) { + if (LicenseUtils.supportedLicense(this.product, this.licenseState)) { return innerPrepareRequest(request, client); } else { // We need to consume parameters and content from the REST request in order to bypass unrecognized param errors diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchInfoTransportAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchInfoTransportAction.java index ecc368791af60..4523a04c9a8c1 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchInfoTransportAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchInfoTransportAction.java @@ -43,7 +43,7 @@ public String name() { @Override public boolean available() { - return LicenseUtils.LICENSED_ENT_SEARCH_FEATURE.checkWithoutTracking(licenseState); + return LicenseUtils.PLATINUM_LICENSED_FEATURE.checkWithoutTracking(licenseState); } @Override diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java index d9b7c325dfd21..4a6a2a3590b3d 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/EnterpriseSearchUsageTransportAction.java @@ -97,7 +97,7 @@ protected void masterOperation( ) { if (enabled == false) { EnterpriseSearchFeatureSetUsage usage = new EnterpriseSearchFeatureSetUsage( - LicenseUtils.LICENSED_ENT_SEARCH_FEATURE.checkWithoutTracking(licenseState), + LicenseUtils.PLATINUM_LICENSED_FEATURE.checkWithoutTracking(licenseState), enabled, Collections.emptyMap(), Collections.emptyMap(), @@ -121,7 +121,7 @@ protected void masterOperation( listener.onResponse( new XPackUsageFeatureResponse( new EnterpriseSearchFeatureSetUsage( - LicenseUtils.LICENSED_ENT_SEARCH_FEATURE.checkWithoutTracking(licenseState), + LicenseUtils.PLATINUM_LICENSED_FEATURE.checkWithoutTracking(licenseState), enabled, searchApplicationsUsage, analyticsCollectionsUsage, @@ -133,7 +133,7 @@ protected void masterOperation( listener.onResponse( new XPackUsageFeatureResponse( new EnterpriseSearchFeatureSetUsage( - LicenseUtils.LICENSED_ENT_SEARCH_FEATURE.checkWithoutTracking(licenseState), + LicenseUtils.PLATINUM_LICENSED_FEATURE.checkWithoutTracking(licenseState), enabled, Collections.emptyMap(), analyticsCollectionsUsage, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java index c531187dbb0a0..b72bffab81e1f 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJob.java @@ -101,7 +101,7 @@ public class ConnectorSyncJob implements Writeable, ToXContentObject { public static final ParseField TRIGGER_METHOD_FIELD = new ParseField("trigger_method"); - static final ParseField WORKER_HOSTNAME_FIELD = new ParseField("worker_hostname"); + public static final ParseField WORKER_HOSTNAME_FIELD = new ParseField("worker_hostname"); static final ConnectorSyncStatus DEFAULT_INITIAL_STATUS = ConnectorSyncStatus.PENDING; diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobConstants.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobConstants.java index cf44ab4e733c8..8dac7c3c30652 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobConstants.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobConstants.java @@ -13,6 +13,7 @@ public class ConnectorSyncJobConstants { public static final String EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE = "[connector_sync_job_id] of the connector sync job cannot be null or empty."; + public static final String EMPTY_WORKER_HOSTNAME_ERROR_MESSAGE = "[worker_hostname] of the connector sync job cannot be null."; public static final String CONNECTOR_SYNC_JOB_ID_PARAM = CONNECTOR_SYNC_JOB_ID_FIELD.getPreferredName(); } diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java index 6d81bae3c2e9f..72ca1f1d8499b 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexService.java @@ -47,7 +47,6 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.application.connector.Connector; import org.elasticsearch.xpack.application.connector.ConnectorFiltering; -import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; import org.elasticsearch.xpack.application.connector.ConnectorTemplateRegistry; import org.elasticsearch.xpack.application.connector.filtering.FilteringRules; @@ -68,6 +67,7 @@ import java.util.stream.Stream; import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.xpack.application.connector.ConnectorIndexService.CONNECTOR_INDEX_NAME; /** * A service that manages persistent {@link ConnectorSyncJob} configurations. @@ -342,7 +342,7 @@ public void listConnectorSyncJobs( String connectorId, ConnectorSyncStatus syncStatus, List jobTypeList, - ActionListener listener + ActionListener listener ) { try { QueryBuilder query = buildListQuery(connectorId, syncStatus, jobTypeList); @@ -368,7 +368,7 @@ public void onResponse(SearchResponse searchResponse) { @Override public void onFailure(Exception e) { if (e instanceof IndexNotFoundException) { - listener.onResponse(new ConnectorSyncJobIndexService.ConnectorSyncJobsResult(Collections.emptyList(), 0L)); + listener.onResponse(new ConnectorSyncJobsResult(Collections.emptyList(), 0L)); return; } listener.onFailure(e); @@ -417,10 +417,7 @@ private ConnectorSyncJobsResult mapSearchResponseToConnectorSyncJobsList(SearchR .map(ConnectorSyncJobIndexService::hitToConnectorSyncJob) .toList(); - return new ConnectorSyncJobIndexService.ConnectorSyncJobsResult( - connectorSyncJobs, - (int) searchResponse.getHits().getTotalHits().value - ); + return new ConnectorSyncJobsResult(connectorSyncJobs, (int) searchResponse.getHits().getTotalHits().value); } private static ConnectorSyncJobSearchResult hitToConnectorSyncJob(SearchHit searchHit) { @@ -497,7 +494,7 @@ private ConnectorSyncStatus getConnectorSyncJobStatusFromSearchResult(ConnectorS private void getSyncJobConnectorInfo(String connectorId, ConnectorSyncJobType jobType, ActionListener listener) { try { - final GetRequest request = new GetRequest(ConnectorIndexService.CONNECTOR_INDEX_NAME, connectorId); + final GetRequest request = new GetRequest(CONNECTOR_INDEX_NAME, connectorId); client.get(request, new ActionListener<>() { @Override @@ -643,6 +640,64 @@ public void deleteAllSyncJobsByConnectorId(String connectorId, ActionListener listener + ) { + + try { + getConnectorSyncJob(connectorSyncJobId, listener.delegateFailure((getSyncJobListener, syncJobSearchResult) -> { + + Map document = new HashMap<>(); + document.put(ConnectorSyncJob.WORKER_HOSTNAME_FIELD.getPreferredName(), workerHostname); + document.put(ConnectorSyncJob.STATUS_FIELD.getPreferredName(), ConnectorSyncStatus.IN_PROGRESS.toString()); + document.put(ConnectorSyncJob.LAST_SEEN_FIELD.getPreferredName(), Instant.now()); + document.put(ConnectorSyncJob.STARTED_AT_FIELD.getPreferredName(), Instant.now()); + + if (syncCursor != null) { + document.put( + ConnectorSyncJob.CONNECTOR_FIELD.getPreferredName(), + Map.of(Connector.SYNC_CURSOR_FIELD.getPreferredName(), syncCursor) + ); + } + + final UpdateRequest updateRequest = new UpdateRequest(CONNECTOR_SYNC_JOB_INDEX_NAME, connectorSyncJobId).setRefreshPolicy( + WriteRequest.RefreshPolicy.IMMEDIATE + ).doc(document); + + client.update( + updateRequest, + new DelegatingIndexNotFoundOrDocumentMissingActionListener<>( + connectorSyncJobId, + listener, + (indexNotFoundListener, updateResponse) -> { + if (updateResponse.getResult() == DocWriteResponse.Result.NOT_FOUND) { + indexNotFoundListener.onFailure(new ResourceNotFoundException(connectorSyncJobId)); + return; + } + indexNotFoundListener.onResponse(updateResponse); + } + + ) + ); + })); + + } catch (Exception e) { + listener.onFailure(e); + } + } + /** * Listeners that checks failures for IndexNotFoundException and DocumentMissingException, * and transforms them in ResourceNotFoundException, invoking onFailure on the delegate listener. diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java new file mode 100644 index 0000000000000..74a7e1bdd0282 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobAction.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.ToXContentObject; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.application.connector.Connector; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJob; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.action.ValidateActions.addValidationError; +import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class ClaimConnectorSyncJobAction { + public static final ParseField CONNECTOR_SYNC_JOB_ID_FIELD = new ParseField("connector_sync_job_id"); + public static final String NAME = "indices:data/write/xpack/connector/sync_job/claim"; + public static final ActionType INSTANCE = new ActionType<>(NAME); + + private ClaimConnectorSyncJobAction() {/* no instances */} + + public static class Request extends ConnectorSyncJobActionRequest implements ToXContentObject { + + private final String connectorSyncJobId; + private final String workerHostname; + private final Object syncCursor; + + public String getConnectorSyncJobId() { + return connectorSyncJobId; + } + + public String getWorkerHostname() { + return workerHostname; + } + + public Object getSyncCursor() { + return syncCursor; + } + + public Request(StreamInput in) throws IOException { + super(in); + this.connectorSyncJobId = in.readString(); + this.workerHostname = in.readString(); + this.syncCursor = in.readGenericValue(); + } + + public Request(String connectorSyncJobId, String workerHostname, Object syncCursor) { + this.connectorSyncJobId = connectorSyncJobId; + this.workerHostname = workerHostname; + this.syncCursor = syncCursor; + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "claim_connector_sync_job", + false, + (args, connectorSyncJobId) -> { + String workerHostname = (String) args[0]; + Object syncCursor = args[1]; + + return new Request(connectorSyncJobId, workerHostname, syncCursor); + } + ); + + static { + PARSER.declareString(constructorArg(), ConnectorSyncJob.WORKER_HOSTNAME_FIELD); + PARSER.declareObject(optionalConstructorArg(), (parser, context) -> parser.map(), Connector.SYNC_CURSOR_FIELD); + } + + public static Request fromXContent(XContentParser parser, String connectorSyncJobId) throws IOException { + return PARSER.parse(parser, connectorSyncJobId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field(ConnectorSyncJob.WORKER_HOSTNAME_FIELD.getPreferredName(), workerHostname); + if (syncCursor != null) { + builder.field(Connector.SYNC_CURSOR_FIELD.getPreferredName(), syncCursor); + } + } + builder.endObject(); + return builder; + } + + public static Request fromXContentBytes(String connectorSyncJobId, BytesReference source, XContentType xContentType) { + try (XContentParser parser = XContentHelper.createParser(XContentParserConfiguration.EMPTY, source, xContentType)) { + return fromXContent(parser, connectorSyncJobId); + } catch (IOException e) { + throw new ElasticsearchParseException("Failed to parse request" + source.utf8ToString()); + } + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = null; + + if (Strings.isNullOrEmpty(connectorSyncJobId)) { + validationException = addValidationError( + ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE, + validationException + ); + } + + if (workerHostname == null) { + validationException = addValidationError( + ConnectorSyncJobConstants.EMPTY_WORKER_HOSTNAME_ERROR_MESSAGE, + validationException + ); + } + + return validationException; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(connectorSyncJobId); + out.writeString(workerHostname); + out.writeGenericValue(syncCursor); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Request request = (Request) o; + return Objects.equals(connectorSyncJobId, request.connectorSyncJobId) + && Objects.equals(workerHostname, request.workerHostname) + && Objects.equals(syncCursor, request.syncCursor); + } + + @Override + public int hashCode() { + return Objects.hash(connectorSyncJobId, workerHostname, syncCursor); + } + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestClaimConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestClaimConnectorSyncJobAction.java new file mode 100644 index 0000000000000..c048f43b6baa6 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/RestClaimConnectorSyncJobAction.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.Scope; +import org.elasticsearch.rest.ServerlessScope; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.application.EnterpriseSearch; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction.CONNECTOR_SYNC_JOB_ID_FIELD; + +@ServerlessScope(Scope.PUBLIC) +public class RestClaimConnectorSyncJobAction extends BaseRestHandler { + private static final String CONNECTOR_SYNC_JOB_ID_PARAM = CONNECTOR_SYNC_JOB_ID_FIELD.getPreferredName(); + + @Override + public String getName() { + return "claim_connector_sync_job_action"; + } + + @Override + public List routes() { + return List.of( + new Route( + RestRequest.Method.PUT, + "/" + EnterpriseSearch.CONNECTOR_SYNC_JOB_API_ENDPOINT + "/{" + CONNECTOR_SYNC_JOB_ID_PARAM + "}/_claim" + ) + ); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + ClaimConnectorSyncJobAction.Request request = ClaimConnectorSyncJobAction.Request.fromXContentBytes( + restRequest.param(CONNECTOR_SYNC_JOB_ID_PARAM), + restRequest.content(), + restRequest.getXContentType() + ); + + return channel -> client.execute(ClaimConnectorSyncJobAction.INSTANCE, request, new RestToXContentListener<>(channel)); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportClaimConnectorSyncJobAction.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportClaimConnectorSyncJobAction.java new file mode 100644 index 0000000000000..8b43e153a06c9 --- /dev/null +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/syncjob/action/TransportClaimConnectorSyncJobAction.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.application.connector.action.ConnectorUpdateActionResponse; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobIndexService; + +public class TransportClaimConnectorSyncJobAction extends HandledTransportAction< + ClaimConnectorSyncJobAction.Request, + ConnectorUpdateActionResponse> { + + protected final ConnectorSyncJobIndexService connectorSyncJobIndexService; + + @Inject + public TransportClaimConnectorSyncJobAction(TransportService transportService, ActionFilters actionFilters, Client client) { + super( + ClaimConnectorSyncJobAction.NAME, + transportService, + actionFilters, + ClaimConnectorSyncJobAction.Request::new, + EsExecutors.DIRECT_EXECUTOR_SERVICE + ); + this.connectorSyncJobIndexService = new ConnectorSyncJobIndexService(client); + } + + @Override + protected void doExecute( + Task task, + ClaimConnectorSyncJobAction.Request request, + ActionListener listener + ) { + connectorSyncJobIndexService.claimConnectorSyncJob( + request.getConnectorSyncJobId(), + request.getWorkerHostname(), + request.getSyncCursor(), + listener.map(r -> new ConnectorUpdateActionResponse(r.getResult())) + ); + } +} diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/utils/LicenseUtils.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/utils/LicenseUtils.java index 63f125067e292..4f4e000f5cd02 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/utils/LicenseUtils.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/utils/LicenseUtils.java @@ -14,43 +14,63 @@ import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.XPackField; +import java.util.Locale; + public final class LicenseUtils { public enum Product { - SEARCH_APPLICATION("search application"), - BEHAVIORAL_ANALYTICS("behavioral analytics"), - QUERY_RULES("query rules"); + SEARCH_APPLICATION("search application", License.OperationMode.PLATINUM), + BEHAVIORAL_ANALYTICS("behavioral analytics", License.OperationMode.PLATINUM), + QUERY_RULES("query rules", License.OperationMode.ENTERPRISE),; private final String name; + private final License.OperationMode requiredLicense; - Product(String name) { + Product(String name, License.OperationMode requiredLicense) { this.name = name; + this.requiredLicense = requiredLicense; } public String getName() { return name; } + + public LicensedFeature.Momentary getLicensedFeature() { + return switch (requiredLicense) { + case PLATINUM -> PLATINUM_LICENSED_FEATURE; + case ENTERPRISE -> ENTERPRISE_LICENSED_FEATURE; + default -> throw new IllegalStateException("Unknown license operation mode: " + requiredLicense); + }; + } } - public static final LicensedFeature.Momentary LICENSED_ENT_SEARCH_FEATURE = LicensedFeature.momentary( + public static final LicensedFeature.Momentary PLATINUM_LICENSED_FEATURE = LicensedFeature.momentary( null, XPackField.ENTERPRISE_SEARCH, License.OperationMode.PLATINUM ); - public static boolean supportedLicense(XPackLicenseState licenseState) { - return LICENSED_ENT_SEARCH_FEATURE.check(licenseState); + public static final LicensedFeature.Momentary ENTERPRISE_LICENSED_FEATURE = LicensedFeature.momentary( + null, + XPackField.ENTERPRISE_SEARCH, + License.OperationMode.ENTERPRISE + ); + + public static boolean supportedLicense(Product product, XPackLicenseState licenseState) { + return product.getLicensedFeature().check(licenseState); } public static ElasticsearchSecurityException newComplianceException(XPackLicenseState licenseState, Product product) { String licenseStatus = licenseState.statusDescription(); + String requiredLicenseStatus = product.requiredLicense.toString().toLowerCase(Locale.ROOT); ElasticsearchSecurityException e = new ElasticsearchSecurityException( "Current license is non-compliant for " + product.getName() + ". Current license is {}. " - + "This feature requires an active trial, platinum or enterprise license.", + + "This feature requires an active trial, {}, or higher license.", RestStatus.FORBIDDEN, - licenseStatus + licenseStatus, + requiredLicenseStatus ); return e; } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchBaseRestHandlerTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchBaseRestHandlerTests.java index 6cf176e21498e..1099603e9be07 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchBaseRestHandlerTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/EnterpriseSearchBaseRestHandlerTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.application; import org.elasticsearch.client.internal.node.NodeClient; +import org.elasticsearch.license.LicensedFeature; import org.elasticsearch.license.MockLicenseState; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.ESTestCase; @@ -16,7 +17,6 @@ import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.xpack.application.utils.LicenseUtils; -import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -26,18 +26,22 @@ public class EnterpriseSearchBaseRestHandlerTests extends ESTestCase { public void testLicenseEnforcement() throws Exception { - MockLicenseState licenseState = MockLicenseState.createMock(); - final LicenseUtils.Product product = LicenseUtils.Product.QUERY_RULES; - final boolean licensedFeature = randomBoolean(); + final boolean isLicensed = randomBoolean(); + MockLicenseState enterpriseLicenseState = mockLicenseState(LicenseUtils.ENTERPRISE_LICENSED_FEATURE, isLicensed); + MockLicenseState platinumLicenseState = mockLicenseState(LicenseUtils.PLATINUM_LICENSED_FEATURE, isLicensed); - when(licenseState.isAllowed(LicenseUtils.LICENSED_ENT_SEARCH_FEATURE)).thenReturn(licensedFeature); - when(licenseState.isActive()).thenReturn(licensedFeature); + testHandler(enterpriseLicenseState, isLicensed); + testHandler(platinumLicenseState, isLicensed); + } + + private void testHandler(MockLicenseState licenseState, boolean isLicensed) throws Exception { + final LicenseUtils.Product product = LicenseUtils.Product.QUERY_RULES; final AtomicBoolean consumerCalled = new AtomicBoolean(false); EnterpriseSearchBaseRestHandler handler = new EnterpriseSearchBaseRestHandler(licenseState, product) { @Override - protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) throws IOException { + protected RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient client) { return channel -> { if (consumerCalled.compareAndSet(false, true) == false) { fail("consumerCalled was not false"); @@ -57,7 +61,7 @@ public List routes() { }; FakeRestRequest fakeRestRequest = new FakeRestRequest(); - FakeRestChannel fakeRestChannel = new FakeRestChannel(fakeRestRequest, randomBoolean(), licensedFeature ? 0 : 1); + FakeRestChannel fakeRestChannel = new FakeRestChannel(fakeRestRequest, randomBoolean(), isLicensed ? 0 : 1); try (var threadPool = createThreadPool()) { final var client = new NoOpNodeClient(threadPool); @@ -65,7 +69,7 @@ public List routes() { verifyNoMoreInteractions(licenseState); handler.handleRequest(fakeRestRequest, fakeRestChannel, client); - if (licensedFeature) { + if (isLicensed) { assertTrue(consumerCalled.get()); assertEquals(0, fakeRestChannel.responses().get()); assertEquals(0, fakeRestChannel.errors().get()); @@ -76,4 +80,12 @@ public List routes() { } } } + + private MockLicenseState mockLicenseState(LicensedFeature licensedFeature, boolean isLicensed) { + MockLicenseState licenseState = MockLicenseState.createMock(); + + when(licenseState.isAllowed(licensedFeature)).thenReturn(isLicensed); + when(licenseState.isActive()).thenReturn(isLicensed); + return licenseState; + } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/AnalyticsTransportActionTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/AnalyticsTransportActionTestUtils.java index 3c75015ca1d6a..c067c165f495d 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/AnalyticsTransportActionTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/analytics/action/AnalyticsTransportActionTestUtils.java @@ -33,7 +33,7 @@ public class AnalyticsTransportActionTestUtils { public static MockLicenseState mockLicenseState(boolean supported) { MockLicenseState licenseState = mock(MockLicenseState.class); - when(licenseState.isAllowed(LicenseUtils.LICENSED_ENT_SEARCH_FEATURE)).thenReturn(supported); + when(licenseState.isAllowed(LicenseUtils.PLATINUM_LICENSED_FEATURE)).thenReturn(supported); when(licenseState.isActive()).thenReturn(supported); when(licenseState.statusDescription()).thenReturn("invalid license"); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java index 5e854bb9d2396..b9a77adc12a3c 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobIndexServiceTests.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.application.connector.ConnectorIndexService; import org.elasticsearch.xpack.application.connector.ConnectorSyncStatus; import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; +import org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.PostConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.UpdateConnectorSyncJobErrorAction; import org.elasticsearch.xpack.application.connector.syncjob.action.UpdateConnectorSyncJobIngestionStatsAction; @@ -887,6 +888,147 @@ public void testTransformConnectorFilteringToSyncJobRepresentation_WithFiltering assertEquals(connectorSyncJobIndexService.transformConnectorFilteringToSyncJobRepresentation(filtering), filtering1.getActive()); } + public void testClaimConnectorSyncJob() throws Exception { + // Create sync job + PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( + connectorOneId + ); + PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); + String syncJobId = response.getId(); + Map syncJobSourceBeforeUpdate = getConnectorSyncJobSourceById(syncJobId); + + @SuppressWarnings("unchecked") + Map syncJobConnectorBeforeUpdate = (Map) syncJobSourceBeforeUpdate.get( + ConnectorSyncJob.CONNECTOR_FIELD.getPreferredName() + ); + + // Claim sync job + ClaimConnectorSyncJobAction.Request claimRequest = new ClaimConnectorSyncJobAction.Request( + syncJobId, + randomAlphaOfLengthBetween(5, 100), + Map.of(randomAlphaOfLengthBetween(5, 100), randomAlphaOfLengthBetween(5, 100)) + ); + UpdateResponse claimResponse = awaitClaimConnectorSyncJob(claimRequest); + Map syncJobSourceAfterUpdate = getConnectorSyncJobSourceById(syncJobId); + @SuppressWarnings("unchecked") + Map syncJobConnectorAfterUpdate = (Map) syncJobSourceAfterUpdate.get( + ConnectorSyncJob.CONNECTOR_FIELD.getPreferredName() + ); + + assertThat(claimResponse.status(), equalTo(RestStatus.OK)); + assertThat(syncJobConnectorAfterUpdate.get("sync_cursor"), equalTo(claimRequest.getSyncCursor())); + assertFieldsDidNotUpdateExceptFieldList( + syncJobConnectorBeforeUpdate, + syncJobConnectorAfterUpdate, + List.of(Connector.SYNC_CURSOR_FIELD) + ); + + assertThat( + syncJobSourceBeforeUpdate.get(ConnectorSyncJob.STATUS_FIELD.getPreferredName()), + equalTo(ConnectorSyncStatus.PENDING.toString()) + ); + assertThat( + syncJobSourceAfterUpdate.get(ConnectorSyncJob.STATUS_FIELD.getPreferredName()), + equalTo(ConnectorSyncStatus.IN_PROGRESS.toString()) + ); + assertFieldsDidNotUpdateExceptFieldList( + syncJobSourceBeforeUpdate, + syncJobSourceAfterUpdate, + List.of( + ConnectorSyncJob.STATUS_FIELD, + ConnectorSyncJob.CONNECTOR_FIELD, + ConnectorSyncJob.LAST_SEEN_FIELD, + ConnectorSyncJob.WORKER_HOSTNAME_FIELD + ) + ); + } + + public void testClaimConnectorSyncJob_WithMissingSyncJobId_ExpectException() { + expectThrows( + ResourceNotFoundException.class, + () -> awaitClaimConnectorSyncJob( + new ClaimConnectorSyncJobAction.Request(NON_EXISTING_SYNC_JOB_ID, randomAlphaOfLengthBetween(5, 100), Map.of()) + ) + ); + } + + public void testClaimConnectorSyncJob_WithMissingSyncCursor() throws Exception { + PostConnectorSyncJobAction.Request syncJobRequest = ConnectorSyncJobTestUtils.getRandomPostConnectorSyncJobActionRequest( + connectorOneId + ); + PostConnectorSyncJobAction.Response response = awaitPutConnectorSyncJob(syncJobRequest); + String syncJobId = response.getId(); + Map syncJobSourceBeforeUpdate = getConnectorSyncJobSourceById(syncJobId); + + @SuppressWarnings("unchecked") + Map syncJobConnectorBeforeUpdate = (Map) syncJobSourceBeforeUpdate.get( + ConnectorSyncJob.CONNECTOR_FIELD.getPreferredName() + ); + + // Claim sync job + ClaimConnectorSyncJobAction.Request claimRequest = new ClaimConnectorSyncJobAction.Request( + syncJobId, + randomAlphaOfLengthBetween(5, 100), + null + ); + + UpdateResponse claimResponse = awaitClaimConnectorSyncJob(claimRequest); + Map syncJobSourceAfterUpdate = getConnectorSyncJobSourceById(syncJobId); + @SuppressWarnings("unchecked") + Map syncJobConnectorAfterUpdate = (Map) syncJobSourceAfterUpdate.get( + ConnectorSyncJob.CONNECTOR_FIELD.getPreferredName() + ); + + assertThat(claimResponse.status(), equalTo(RestStatus.OK)); + assertThat(syncJobConnectorAfterUpdate.get("sync_cursor"), nullValue()); + assertThat(syncJobConnectorBeforeUpdate, equalTo(syncJobConnectorAfterUpdate)); + assertFieldsDidNotUpdateExceptFieldList( + syncJobSourceBeforeUpdate, + syncJobSourceAfterUpdate, + List.of(ConnectorSyncJob.STATUS_FIELD, ConnectorSyncJob.LAST_SEEN_FIELD, ConnectorSyncJob.WORKER_HOSTNAME_FIELD) + ); + + assertThat( + syncJobSourceBeforeUpdate.get(ConnectorSyncJob.STATUS_FIELD.getPreferredName()), + equalTo(ConnectorSyncStatus.PENDING.toString()) + ); + assertThat( + syncJobSourceAfterUpdate.get(ConnectorSyncJob.STATUS_FIELD.getPreferredName()), + equalTo(ConnectorSyncStatus.IN_PROGRESS.toString()) + ); + + } + + private UpdateResponse awaitClaimConnectorSyncJob(ClaimConnectorSyncJobAction.Request request) throws Exception { + CountDownLatch latch = new CountDownLatch(1); + final AtomicReference resp = new AtomicReference<>(null); + final AtomicReference exc = new AtomicReference<>(null); + connectorSyncJobIndexService.claimConnectorSyncJob( + request.getConnectorSyncJobId(), + request.getWorkerHostname(), + request.getSyncCursor(), + new ActionListener<>() { + @Override + public void onResponse(UpdateResponse updateResponse) { + resp.set(updateResponse); + latch.countDown(); + } + + @Override + public void onFailure(Exception e) { + exc.set(e); + latch.countDown(); + } + } + ); + assertTrue("Timeout waiting for claim request", latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + if (exc.get() != null) { + throw exc.get(); + } + assertNotNull("Received null response from claim request", resp.get()); + return resp.get(); + } + private UpdateResponse awaitUpdateConnectorSyncJobIngestionStats(UpdateConnectorSyncJobIngestionStatsAction.Request request) throws Exception { CountDownLatch latch = new CountDownLatch(1); diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java index dcc6c9ba242d6..a4ff76e6f2cf9 100644 --- a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/ConnectorSyncJobTestUtils.java @@ -15,6 +15,7 @@ import org.elasticsearch.xpack.application.connector.ConnectorTestUtils; import org.elasticsearch.xpack.application.connector.syncjob.action.CancelConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.CheckInConnectorSyncJobAction; +import org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.DeleteConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.GetConnectorSyncJobAction; import org.elasticsearch.xpack.application.connector.syncjob.action.ListConnectorSyncJobsAction; @@ -29,6 +30,7 @@ import static org.elasticsearch.test.ESTestCase.randomAlphaOfLength; import static org.elasticsearch.test.ESTestCase.randomAlphaOfLengthBetween; +import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomFrom; import static org.elasticsearch.test.ESTestCase.randomInstantBetween; import static org.elasticsearch.test.ESTestCase.randomInt; @@ -194,4 +196,12 @@ public static ListConnectorSyncJobsAction.Request getRandomListConnectorSyncJobs Collections.singletonList(ConnectorTestUtils.getRandomSyncJobType()) ); } + + public static ClaimConnectorSyncJobAction.Request getRandomClaimConnectorSyncJobActionRequest() { + return new ClaimConnectorSyncJobAction.Request( + randomAlphaOfLength(10), + randomAlphaOfLengthBetween(10, 100), + randomBoolean() ? Map.of("test", "123") : null + ); + } } diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionRequestBWCSerializingTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionRequestBWCSerializingTests.java new file mode 100644 index 0000000000000..4a3dc96bafc8a --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionRequestBWCSerializingTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobTestUtils; +import org.elasticsearch.xpack.core.ml.AbstractBWCSerializationTestCase; + +import java.io.IOException; + +public class ClaimConnectorSyncJobActionRequestBWCSerializingTests extends AbstractBWCSerializationTestCase< + ClaimConnectorSyncJobAction.Request> { + + public String connectorSyncJobId; + + @Override + protected Writeable.Reader instanceReader() { + return ClaimConnectorSyncJobAction.Request::new; + } + + @Override + protected ClaimConnectorSyncJobAction.Request createTestInstance() { + ClaimConnectorSyncJobAction.Request request = ConnectorSyncJobTestUtils.getRandomClaimConnectorSyncJobActionRequest(); + connectorSyncJobId = request.getConnectorSyncJobId(); + return request; + } + + @Override + protected ClaimConnectorSyncJobAction.Request mutateInstance(ClaimConnectorSyncJobAction.Request instance) throws IOException { + return randomValueOtherThan(instance, this::createTestInstance); + } + + protected ClaimConnectorSyncJobAction.Request doParseInstance(XContentParser parser) throws IOException { + return ClaimConnectorSyncJobAction.Request.fromXContent(parser, connectorSyncJobId); + } + + @Override + protected ClaimConnectorSyncJobAction.Request mutateInstanceForVersion( + ClaimConnectorSyncJobAction.Request instance, + TransportVersion version + ) { + return new ClaimConnectorSyncJobAction.Request( + instance.getConnectorSyncJobId(), + instance.getWorkerHostname(), + instance.getSyncCursor() + ); + } + +} diff --git a/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionTests.java b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionTests.java new file mode 100644 index 0000000000000..fb6f6280c1098 --- /dev/null +++ b/x-pack/plugin/ent-search/src/test/java/org/elasticsearch/xpack/application/connector/syncjob/action/ClaimConnectorSyncJobActionTests.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.application.connector.syncjob.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.application.connector.syncjob.ConnectorSyncJobConstants; + +import java.util.Collections; + +import static org.elasticsearch.xpack.application.connector.syncjob.action.ClaimConnectorSyncJobAction.Request; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; + +public class ClaimConnectorSyncJobActionTests extends ESTestCase { + + public void testValidate_WhenAllFieldsArePresent_ExpectNoValidationError() { + Request request = new Request(randomAlphaOfLength(10), randomAlphaOfLengthBetween(10, 100), Collections.emptyMap()); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, nullValue()); + } + + public void testValidate_WhenCursorIsNull_ExpectNoValidationError() { + Request request = new Request(randomAlphaOfLength(10), randomAlphaOfLengthBetween(10, 100), null); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, nullValue()); + } + + public void testValidate_WhenConnectorSyncJobIdIsEmpty_ExpectValidationError() { + Request request = new Request("", randomAlphaOfLengthBetween(10, 100), null); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } + + public void testValidate_WhenConnectorSyncJobIdIsNull_ExpectValidationError() { + Request request = new Request(null, randomAlphaOfLengthBetween(10, 100), null); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_CONNECTOR_SYNC_JOB_ID_ERROR_MESSAGE)); + } + + public void testValidate_WhenWorkerHostnameIsNull_ExpectValidationError() { + Request request = new Request(randomAlphaOfLength(10), null, null); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, notNullValue()); + assertThat(exception.getMessage(), containsString(ConnectorSyncJobConstants.EMPTY_WORKER_HOSTNAME_ERROR_MESSAGE)); + } + + public void testValidate_WhenSyncCursorIsEmptyObject_ExpectNoError() { + Request request = new Request(randomAlphaOfLength(10), randomAlphaOfLength(10), Collections.emptyMap()); + ActionRequestValidationException exception = request.validate(); + + assertThat(exception, nullValue()); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsRequest.java index 53e9e1d1a0137..56335be32de6c 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsRequest.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.eql.plugin; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -34,11 +33,6 @@ public void includeStats(boolean includeStats) { this.includeStats = includeStats; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - @Override public String toString() { return "eql_stats"; diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index a6e713007a97f..0f7d92564c8ab 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -168,12 +168,14 @@ protected Attribute clone( @Override public int hashCode() { - return Objects.hash(super.hashCode(), path); + return Objects.hash(super.hashCode(), path, field); } @Override public boolean equals(Object obj) { - return super.equals(obj) && Objects.equals(path, ((FieldAttribute) obj).path); + return super.equals(obj) + && Objects.equals(path, ((FieldAttribute) obj).path) + && Objects.equals(field, ((FieldAttribute) obj).field); } @Override diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java index 68780f5b32e9c..20cdbaf6acdbf 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/Literal.java @@ -6,17 +6,35 @@ */ package org.elasticsearch.xpack.esql.core.expression; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; +import java.io.IOException; +import java.util.List; import java.util.Objects; +import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; +import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; +import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; + /** - * SQL Literal or constant. + * Literal or constant. */ public class Literal extends LeafExpression { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "Literal", + Literal::readFrom + ); public static final Literal TRUE = new Literal(Source.EMPTY, Boolean.TRUE, DataType.BOOLEAN); public static final Literal FALSE = new Literal(Source.EMPTY, Boolean.FALSE, DataType.BOOLEAN); @@ -31,6 +49,25 @@ public Literal(Source source, Object value, DataType dataType) { this.value = value; } + private static Literal readFrom(StreamInput in) throws IOException { + Source source = Source.readFrom((StreamInput & PlanStreamInput) in); + Object value = in.readGenericValue(); + DataType dataType = DataType.readFrom(in); + return new Literal(source, mapToLiteralValue(in, dataType, value), dataType); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + out.writeGenericValue(mapFromLiteralValue(out, dataType, value)); + dataType.writeTo(out); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, Literal::new, value, dataType); @@ -112,4 +149,50 @@ public static Literal of(Expression foldable) { public static Literal of(Expression source, Object value) { return new Literal(source.source(), value, source.dataType()); } + + /** + * Not all literal values are currently supported in StreamInput/StreamOutput as generic values. + * This mapper allows for addition of new and interesting values without (yet) adding to StreamInput/Output. + * This makes the most sense during the pre-GA version of ESQL. When we get near GA we might want to push this down. + *

    + * For the spatial point type support we need to care about the fact that 8.12.0 uses encoded longs for serializing + * while 8.13 uses WKB. + */ + private static Object mapFromLiteralValue(StreamOutput out, DataType dataType, Object value) { + if (dataType == GEO_POINT || dataType == CARTESIAN_POINT) { + // In 8.12.0 we serialized point literals as encoded longs, but now use WKB + if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) { + if (value instanceof List list) { + return list.stream().map(v -> mapFromLiteralValue(out, dataType, v)).toList(); + } + return wkbAsLong(dataType, (BytesRef) value); + } + } + return value; + } + + /** + * Not all literal values are currently supported in StreamInput/StreamOutput as generic values. + * This mapper allows for addition of new and interesting values without (yet) changing StreamInput/Output. + */ + private static Object mapToLiteralValue(StreamInput in, DataType dataType, Object value) { + if (dataType == GEO_POINT || dataType == CARTESIAN_POINT) { + // In 8.12.0 we serialized point literals as encoded longs, but now use WKB + if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) { + if (value instanceof List list) { + return list.stream().map(v -> mapToLiteralValue(in, dataType, v)).toList(); + } + return longAsWKB(dataType, (Long) value); + } + } + return value; + } + + private static BytesRef longAsWKB(DataType dataType, long encoded) { + return dataType == GEO_POINT ? GEO.longAsWkb(encoded) : CARTESIAN.longAsWkb(encoded); + } + + private static long wkbAsLong(DataType dataType, BytesRef wkb) { + return dataType == GEO_POINT ? GEO.wkbAsLong(wkb) : CARTESIAN.wkbAsLong(wkb); + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/BinaryScalarFunction.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/BinaryScalarFunction.java index f96aeb693b52a..4b462719a375b 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/BinaryScalarFunction.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/BinaryScalarFunction.java @@ -6,9 +6,14 @@ */ package org.elasticsearch.xpack.esql.core.expression.function.scalar; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; +import java.io.IOException; import java.util.Arrays; import java.util.List; @@ -22,6 +27,21 @@ protected BinaryScalarFunction(Source source, Expression left, Expression right) this.right = right; } + protected BinaryScalarFunction(StreamInput in) throws IOException { + this( + Source.readFrom((StreamInput & PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(left()); + ((PlanStreamOutput) out).writeExpression(right()); + } + @Override public final BinaryScalarFunction replaceChildren(List newChildren) { Expression newLeft = newChildren.get(0); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java index 2ef0b892138de..e5c2cedfd087b 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/UnaryScalarFunction.java @@ -6,10 +6,15 @@ */ package org.elasticsearch.xpack.esql.core.expression.function.scalar; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; +import java.io.IOException; import java.util.List; import static java.util.Collections.singletonList; @@ -18,16 +23,21 @@ public abstract class UnaryScalarFunction extends ScalarFunction { private final Expression field; - protected UnaryScalarFunction(Source source) { - super(source); - this.field = null; - } - protected UnaryScalarFunction(Source source, Expression field) { super(source, singletonList(field)); this.field = field; } + protected UnaryScalarFunction(StreamInput in) throws IOException { + this(Source.readFrom((StreamInput & PlanStreamInput) in), ((PlanStreamInput) in).readExpression()); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(field); + } + @Override public final UnaryScalarFunction replaceChildren(List newChildren) { return replaceChild(newChildren.get(0)); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java index 8da858865ed3f..e8ca84bc72988 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/FullTextPredicate.java @@ -6,17 +6,28 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamInput; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Objects; public abstract class FullTextPredicate extends Expression { + public static List getNamedWriteables() { + return List.of(MatchQueryPredicate.ENTRY, MultiMatchQueryPredicate.ENTRY, StringQueryPredicate.ENTRY); + } + public enum Operator { AND, OR; @@ -32,7 +43,7 @@ public org.elasticsearch.index.query.Operator toEs() { // common properties private final String analyzer; - FullTextPredicate(Source source, String query, String options, List children) { + FullTextPredicate(Source source, String query, @Nullable String options, List children) { super(source, children); this.query = query; this.options = options; @@ -41,6 +52,15 @@ public org.elasticsearch.index.query.Operator toEs() { this.analyzer = optionMap.get("analyzer"); } + protected FullTextPredicate(StreamInput in) throws IOException { + this( + Source.readFrom((StreamInput & PlanStreamInput) in), + in.readString(), + in.readOptionalString(), + in.readCollectionAsList(input -> ((PlanStreamInput) in).readExpression()) + ); + } + public String query() { return query; } @@ -67,6 +87,14 @@ public DataType dataType() { return DataType.BOOLEAN; } + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + out.writeString(query); + out.writeOptionalString(options); + out.writeCollection(children(), (o, v) -> ((PlanStreamOutput) o).writeExpression(v)); + } + @Override public int hashCode() { return Objects.hash(query, options); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MatchQueryPredicate.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MatchQueryPredicate.java index fc5bd6320e445..f2e6088167ba5 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MatchQueryPredicate.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MatchQueryPredicate.java @@ -6,10 +6,13 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import java.io.IOException; import java.util.List; import java.util.Objects; @@ -17,6 +20,12 @@ public class MatchQueryPredicate extends FullTextPredicate { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "MatchQueryPredicate", + MatchQueryPredicate::new + ); + private final Expression field; public MatchQueryPredicate(Source source, Expression field, String query, String options) { @@ -24,6 +33,12 @@ public MatchQueryPredicate(Source source, Expression field, String query, String this.field = field; } + MatchQueryPredicate(StreamInput in) throws IOException { + super(in); + assert super.children().size() == 1; + field = super.children().get(0); + } + @Override protected NodeInfo info() { return NodeInfo.create(this, MatchQueryPredicate::new, field, query(), options()); @@ -51,4 +66,9 @@ public boolean equals(Object obj) { } return false; } + + @Override + public String getWriteableName() { + return ENTRY.name; + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MultiMatchQueryPredicate.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MultiMatchQueryPredicate.java index 9e9d55ab4759a..2d66023a1407d 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MultiMatchQueryPredicate.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/MultiMatchQueryPredicate.java @@ -6,10 +6,14 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.Objects; @@ -18,6 +22,12 @@ public class MultiMatchQueryPredicate extends FullTextPredicate { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "MultiMatchQueryPredicate", + MultiMatchQueryPredicate::new + ); + private final String fieldString; private final Map fields; @@ -28,6 +38,14 @@ public MultiMatchQueryPredicate(Source source, String fieldString, String query, this.fields = FullTextUtils.parseFields(fieldString, source); } + MultiMatchQueryPredicate(StreamInput in) throws IOException { + super(in); + assert super.children().isEmpty(); + fieldString = in.readString(); + // inferred + this.fields = FullTextUtils.parseFields(fieldString, source()); + } + @Override protected NodeInfo info() { return NodeInfo.create(this, MultiMatchQueryPredicate::new, fieldString, query(), options()); @@ -46,6 +64,12 @@ public Map fields() { return fields; } + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(fieldString); + } + @Override public int hashCode() { return Objects.hash(fieldString, super.hashCode()); @@ -59,4 +83,9 @@ public boolean equals(Object obj) { } return false; } + + @Override + public String getWriteableName() { + return ENTRY.name; + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/StringQueryPredicate.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/StringQueryPredicate.java index 17b673cb0da4e..95000a5364e12 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/StringQueryPredicate.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/fulltext/StringQueryPredicate.java @@ -6,10 +6,13 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.fulltext; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import java.io.IOException; import java.util.List; import java.util.Map; @@ -17,6 +20,12 @@ public final class StringQueryPredicate extends FullTextPredicate { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "StringQueryPredicate", + StringQueryPredicate::new + ); + private final Map fields; public StringQueryPredicate(Source source, String query, String options) { @@ -26,6 +35,12 @@ public StringQueryPredicate(Source source, String query, String options) { this.fields = FullTextUtils.parseFields(optionMap(), source); } + StringQueryPredicate(StreamInput in) throws IOException { + super(in); + assert super.children().isEmpty(); + this.fields = FullTextUtils.parseFields(optionMap(), source()); + } + @Override protected NodeInfo info() { return NodeInfo.create(this, StringQueryPredicate::new, query(), options()); @@ -39,4 +54,9 @@ public Expression replaceChildren(List newChildren) { public Map fields() { return fields; } + + @Override + public String getWriteableName() { + return ENTRY.name; + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java index 31c63393afaea..5f183a1cc26ea 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/logical/Not.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.logical; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; import org.elasticsearch.xpack.esql.core.expression.gen.processor.Processor; @@ -14,15 +16,27 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import java.io.IOException; + import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isBoolean; public class Not extends UnaryScalarFunction implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Not", Not::new); public Not(Source source, Expression child) { super(source, child); } + private Not(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, Not::new, field()); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java index 52375c5db01a1..e365480a6fd79 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNotNull.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.nulls; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; @@ -16,12 +18,28 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import java.io.IOException; + public class IsNotNull extends UnaryScalarFunction implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "IsNotNull", + IsNotNull::new + ); public IsNotNull(Source source, Expression field) { super(source, field); } + private IsNotNull(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, IsNotNull::new, field()); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java index d52eec9114df6..8b6eb5d4404b0 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/predicate/nulls/IsNull.java @@ -6,6 +6,8 @@ */ package org.elasticsearch.xpack.esql.core.expression.predicate.nulls; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction; @@ -16,12 +18,24 @@ import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import java.io.IOException; + public class IsNull extends UnaryScalarFunction implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "IsNull", IsNull::new); public IsNull(Source source, Expression field) { super(source, field); } + private IsNull(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, IsNull::new, field()); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexCompatibility.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexCompatibility.java deleted file mode 100644 index 6cc0816661f01..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexCompatibility.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.core.index; - -import org.elasticsearch.Version; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; - -import java.util.Map; - -import static org.elasticsearch.xpack.esql.core.index.VersionCompatibilityChecks.isTypeSupportedInVersion; -import static org.elasticsearch.xpack.esql.core.type.DataType.isPrimitive; -import static org.elasticsearch.xpack.esql.core.type.Types.propagateUnsupportedType; - -public final class IndexCompatibility { - - public static Map compatible(Map mapping, Version version) { - for (Map.Entry entry : mapping.entrySet()) { - EsField esField = entry.getValue(); - DataType dataType = esField.getDataType(); - if (isPrimitive(dataType) == false) { - compatible(esField.getProperties(), version); - } else if (isTypeSupportedInVersion(dataType, version) == false) { - EsField field = new UnsupportedEsField(entry.getKey(), dataType.nameUpper(), null, esField.getProperties()); - entry.setValue(field); - propagateUnsupportedType(entry.getKey(), dataType.nameUpper(), esField.getProperties()); - } - } - return mapping; - } - - public static EsIndex compatible(EsIndex esIndex, Version version) { - compatible(esIndex.mapping(), version); - return esIndex; - } - - public static IndexResolution compatible(IndexResolution indexResolution, Version version) { - if (indexResolution.isValid()) { - compatible(indexResolution.get(), version); - } - return indexResolution; - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolver.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolver.java deleted file mode 100644 index 63467eaadd8df..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/IndexResolver.java +++ /dev/null @@ -1,1046 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.esql.core.index; - -import org.elasticsearch.ElasticsearchSecurityException; -import org.elasticsearch.action.ActionListener; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest; -import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature; -import org.elasticsearch.action.admin.indices.resolve.ResolveIndexAction; -import org.elasticsearch.action.fieldcaps.FieldCapabilities; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; -import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.client.internal.Client; -import org.elasticsearch.cluster.metadata.AliasMetadata; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.mapper.TimeSeriesParams; -import org.elasticsearch.transport.NoSuchRemoteClusterException; -import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.DataTypeRegistry; -import org.elasticsearch.xpack.esql.core.type.DateEsField; -import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; -import org.elasticsearch.xpack.esql.core.type.KeywordEsField; -import org.elasticsearch.xpack.esql.core.type.TextEsField; -import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; -import org.elasticsearch.xpack.esql.core.util.CollectionUtils; -import org.elasticsearch.xpack.esql.core.util.Holder; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.regex.Pattern; - -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static org.elasticsearch.action.ActionListener.wrap; -import static org.elasticsearch.common.Strings.hasText; -import static org.elasticsearch.common.regex.Regex.simpleMatch; -import static org.elasticsearch.transport.RemoteClusterAware.buildRemoteIndexName; -import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; -import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; -import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; -import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; -import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; -import static org.elasticsearch.xpack.esql.core.util.StringUtils.qualifyAndJoinIndices; -import static org.elasticsearch.xpack.esql.core.util.StringUtils.splitQualifiedIndex; - -public class IndexResolver { - - public enum IndexType { - STANDARD_INDEX(SQL_TABLE, "INDEX"), - ALIAS(SQL_VIEW, "ALIAS"), - FROZEN_INDEX(SQL_TABLE, "FROZEN INDEX"), - // value for user types unrecognized - UNKNOWN("UNKNOWN", "UNKNOWN"); - - public static final EnumSet VALID_INCLUDE_FROZEN = EnumSet.of(STANDARD_INDEX, ALIAS, FROZEN_INDEX); - public static final EnumSet VALID_REGULAR = EnumSet.of(STANDARD_INDEX, ALIAS); - - private final String toSql; - private final String toNative; - - IndexType(String sql, String toNative) { - this.toSql = sql; - this.toNative = toNative; - } - - public String toSql() { - return toSql; - } - - public String toNative() { - return toNative; - } - } - - public record IndexInfo(String cluster, String name, IndexType type) { - - @Override - public String toString() { - return buildRemoteIndexName(cluster, name); - } - - } - - public static final String SQL_TABLE = "TABLE"; - public static final String SQL_VIEW = "VIEW"; - - private static final IndicesOptions INDICES_ONLY_OPTIONS = IndicesOptions.builder() - .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) - .wildcardOptions( - IndicesOptions.WildcardOptions.builder() - .matchOpen(true) - .matchClosed(false) - .includeHidden(false) - .allowEmptyExpressions(true) - .resolveAliases(false) - ) - .gatekeeperOptions( - IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).allowClosedIndices(true).allowAliasToMultipleIndices(true) - ) - .build(); - private static final IndicesOptions FROZEN_INDICES_OPTIONS = IndicesOptions.builder() - .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) - .wildcardOptions( - IndicesOptions.WildcardOptions.builder() - .matchOpen(true) - .matchClosed(false) - .includeHidden(false) - .allowEmptyExpressions(true) - .resolveAliases(false) - ) - .gatekeeperOptions( - IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(false).allowClosedIndices(true).allowAliasToMultipleIndices(true) - ) - .build(); - - public static final IndicesOptions FIELD_CAPS_INDICES_OPTIONS = IndicesOptions.builder() - .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) - .wildcardOptions( - IndicesOptions.WildcardOptions.builder() - .matchOpen(true) - .matchClosed(false) - .includeHidden(false) - .allowEmptyExpressions(true) - .resolveAliases(true) - ) - .gatekeeperOptions( - IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).allowClosedIndices(true).allowAliasToMultipleIndices(true) - ) - .build(); - public static final IndicesOptions FIELD_CAPS_FROZEN_INDICES_OPTIONS = IndicesOptions.builder() - .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) - .wildcardOptions( - IndicesOptions.WildcardOptions.builder() - .matchOpen(true) - .matchClosed(false) - .includeHidden(false) - .allowEmptyExpressions(true) - .resolveAliases(true) - ) - .gatekeeperOptions( - IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(false).allowClosedIndices(true).allowAliasToMultipleIndices(true) - ) - .build(); - - public static final Set ALL_FIELDS = Set.of("*"); - public static final Set INDEX_METADATA_FIELD = Set.of("_index"); - public static final String UNMAPPED = "unmapped"; - - private final Client client; - private final String clusterName; - private final DataTypeRegistry typeRegistry; - - private final Supplier> remoteClusters; - - public IndexResolver(Client client, String clusterName, DataTypeRegistry typeRegistry, Supplier> remoteClusters) { - this.client = client; - this.clusterName = clusterName; - this.typeRegistry = typeRegistry; - this.remoteClusters = remoteClusters; - } - - public String clusterName() { - return clusterName; - } - - public Set remoteClusters() { - return remoteClusters.get(); - } - - /** - * Resolves only the names, differentiating between indices and aliases. - * This method is required since the other methods rely on mapping which is tied to an index (not an alias). - */ - public void resolveNames( - String clusterWildcard, - String indexWildcard, - String javaRegex, - EnumSet types, - ActionListener> listener - ) { - - // first get aliases (if specified) - boolean retrieveAliases = CollectionUtils.isEmpty(types) || types.contains(IndexType.ALIAS); - boolean retrieveIndices = CollectionUtils.isEmpty(types) || types.contains(IndexType.STANDARD_INDEX); - boolean retrieveFrozenIndices = CollectionUtils.isEmpty(types) || types.contains(IndexType.FROZEN_INDEX); - - String[] indexWildcards = Strings.commaDelimitedListToStringArray(indexWildcard); - Set indexInfos = new HashSet<>(); - if (retrieveAliases && clusterIsLocal(clusterWildcard)) { - ResolveIndexAction.Request resolveRequest = new ResolveIndexAction.Request(indexWildcards, IndicesOptions.lenientExpandOpen()); - client.admin().indices().resolveIndex(resolveRequest, wrap(response -> { - for (ResolveIndexAction.ResolvedAlias alias : response.getAliases()) { - indexInfos.add(new IndexInfo(clusterName, alias.getName(), IndexType.ALIAS)); - } - for (ResolveIndexAction.ResolvedDataStream dataStream : response.getDataStreams()) { - indexInfos.add(new IndexInfo(clusterName, dataStream.getName(), IndexType.ALIAS)); - } - resolveIndices(clusterWildcard, indexWildcards, javaRegex, retrieveIndices, retrieveFrozenIndices, indexInfos, listener); - }, ex -> { - // with security, two exception can be thrown: - // INFE - if no alias matches - // security exception is the user cannot access aliases - - // in both cases, that is allowed and we continue with the indices request - if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - resolveIndices( - clusterWildcard, - indexWildcards, - javaRegex, - retrieveIndices, - retrieveFrozenIndices, - indexInfos, - listener - ); - } else { - listener.onFailure(ex); - } - })); - } else { - resolveIndices(clusterWildcard, indexWildcards, javaRegex, retrieveIndices, retrieveFrozenIndices, indexInfos, listener); - } - } - - private void resolveIndices( - String clusterWildcard, - String[] indexWildcards, - String javaRegex, - boolean retrieveIndices, - boolean retrieveFrozenIndices, - Set indexInfos, - ActionListener> listener - ) { - if (retrieveIndices || retrieveFrozenIndices) { - if (clusterIsLocal(clusterWildcard)) { // resolve local indices - GetIndexRequest indexRequest = new GetIndexRequest().local(true) - .indices(indexWildcards) - .features(Feature.SETTINGS) - .includeDefaults(false) - .indicesOptions(INDICES_ONLY_OPTIONS); - - // if frozen indices are requested, make sure to update the request accordingly - if (retrieveFrozenIndices) { - indexRequest.indicesOptions(FROZEN_INDICES_OPTIONS); - } - - client.admin().indices().getIndex(indexRequest, listener.delegateFailureAndWrap((delegate, indices) -> { - if (indices != null) { - for (String indexName : indices.getIndices()) { - boolean isFrozen = retrieveFrozenIndices - && indices.getSettings().get(indexName).getAsBoolean("index.frozen", false); - indexInfos.add( - new IndexInfo(clusterName, indexName, isFrozen ? IndexType.FROZEN_INDEX : IndexType.STANDARD_INDEX) - ); - } - } - resolveRemoteIndices(clusterWildcard, indexWildcards, javaRegex, retrieveFrozenIndices, indexInfos, delegate); - })); - } else { - resolveRemoteIndices(clusterWildcard, indexWildcards, javaRegex, retrieveFrozenIndices, indexInfos, listener); - } - } else { - filterResults(javaRegex, indexInfos, listener); - } - } - - private void resolveRemoteIndices( - String clusterWildcard, - String[] indexWildcards, - String javaRegex, - boolean retrieveFrozenIndices, - Set indexInfos, - ActionListener> listener - ) { - if (hasText(clusterWildcard)) { - IndicesOptions indicesOptions = retrieveFrozenIndices ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS; - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest( - qualifyAndJoinIndices(clusterWildcard, indexWildcards), - ALL_FIELDS, - indicesOptions, - emptyMap() - ); - client.fieldCaps(fieldRequest, wrap(response -> { - String[] indices = response.getIndices(); - if (indices != null) { - for (String indexName : indices) { - // TODO: perform two requests w/ & w/o frozen option to retrieve (by diff) the throttling status? - Tuple splitRef = splitQualifiedIndex(indexName); - // Field caps on "remote:foo" should always return either empty or remote indices. But in case cluster's - // detail is missing, it's going to be a local index. TODO: why would this happen? - String cluster = splitRef.v1() == null ? clusterName : splitRef.v1(); - indexInfos.add(new IndexInfo(cluster, splitRef.v2(), IndexType.STANDARD_INDEX)); - } - } - filterResults(javaRegex, indexInfos, listener); - }, ex -> { - // see comment in resolveNames() - if (ex instanceof NoSuchRemoteClusterException || ex instanceof ElasticsearchSecurityException) { - filterResults(javaRegex, indexInfos, listener); - } else { - listener.onFailure(ex); - } - })); - } else { - filterResults(javaRegex, indexInfos, listener); - } - } - - private static void filterResults(String javaRegex, Set indexInfos, ActionListener> listener) { - - // since the index name does not support ?, filter the results manually - Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null; - - Set result = new TreeSet<>(Comparator.comparing(IndexInfo::cluster).thenComparing(IndexInfo::name)); - for (IndexInfo indexInfo : indexInfos) { - if (pattern == null || pattern.matcher(indexInfo.name()).matches()) { - result.add(indexInfo); - } - } - listener.onResponse(result); - } - - private boolean clusterIsLocal(String clusterWildcard) { - return clusterWildcard == null || simpleMatch(clusterWildcard, clusterName); - } - - /** - * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. - */ - public void resolveAsMergedMapping( - String indexWildcard, - Set fieldNames, - IndicesOptions indicesOptions, - Map runtimeMappings, - ActionListener listener - ) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, fieldNames, indicesOptions, runtimeMappings); - client.fieldCaps( - fieldRequest, - listener.delegateFailureAndWrap((l, response) -> l.onResponse(mergedMappings(typeRegistry, indexWildcard, response))) - ); - } - - /** - * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. - */ - public void resolveAsMergedMapping( - String indexWildcard, - Set fieldNames, - boolean includeFrozen, - Map runtimeMappings, - ActionListener listener - ) { - resolveAsMergedMapping(indexWildcard, fieldNames, includeFrozen, runtimeMappings, listener, (fieldName, types) -> null); - } - - /** - * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. - */ - public void resolveAsMergedMapping( - String indexWildcard, - Set fieldNames, - boolean includeFrozen, - Map runtimeMappings, - ActionListener listener, - BiFunction, InvalidMappedField> specificValidityVerifier - ) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, fieldNames, includeFrozen, runtimeMappings); - client.fieldCaps( - fieldRequest, - listener.delegateFailureAndWrap( - (l, response) -> l.onResponse(mergedMappings(typeRegistry, indexWildcard, response, specificValidityVerifier, null, null)) - ) - ); - } - - public void resolveAsMergedMapping( - String indexWildcard, - Set fieldNames, - boolean includeFrozen, - Map runtimeMappings, - ActionListener listener, - BiFunction, InvalidMappedField> specificValidityVerifier, - BiConsumer fieldUpdater, - Set allowedMetadataFields - ) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, fieldNames, includeFrozen, runtimeMappings); - client.fieldCaps( - fieldRequest, - listener.delegateFailureAndWrap( - (l, response) -> l.onResponse( - mergedMappings(typeRegistry, indexWildcard, response, specificValidityVerifier, fieldUpdater, allowedMetadataFields) - ) - ) - ); - } - - public static IndexResolution mergedMappings( - DataTypeRegistry typeRegistry, - String indexPattern, - FieldCapabilitiesResponse fieldCapsResponse, - BiFunction, InvalidMappedField> specificValidityVerifier - ) { - return mergedMappings(typeRegistry, indexPattern, fieldCapsResponse, specificValidityVerifier, null, null); - } - - public static IndexResolution mergedMappings( - DataTypeRegistry typeRegistry, - String indexPattern, - FieldCapabilitiesResponse fieldCapsResponse, - BiFunction, InvalidMappedField> specificValidityVerifier, - BiConsumer fieldUpdater, - Set allowedMetadataFields - ) { - - if (fieldCapsResponse.getIndices().length == 0) { - return IndexResolution.notFound(indexPattern); - } - - BiFunction, InvalidMappedField> validityVerifier = (fieldName, types) -> { - InvalidMappedField f = specificValidityVerifier.apply(fieldName, types); - if (f != null) { - return f; - } - - StringBuilder errorMessage = new StringBuilder(); - boolean hasUnmapped = types.containsKey(UNMAPPED); - - if (types.size() > (hasUnmapped ? 2 : 1)) { - // build the error message - // and create a MultiTypeField - - for (Entry type : types.entrySet()) { - // skip unmapped - if (UNMAPPED.equals(type.getKey())) { - continue; - } - - if (errorMessage.length() > 0) { - errorMessage.append(", "); - } - errorMessage.append("["); - errorMessage.append(type.getKey()); - errorMessage.append("] in "); - errorMessage.append(Arrays.toString(type.getValue().indices())); - } - - errorMessage.insert(0, "mapped as [" + (types.size() - (hasUnmapped ? 1 : 0)) + "] incompatible types: "); - - return new InvalidMappedField(fieldName, errorMessage.toString()); - } - // type is okay, check aggregation - else { - FieldCapabilities fieldCap = types.values().iterator().next(); - - // validate search/agg-able - if (fieldCap.isAggregatable() && fieldCap.nonAggregatableIndices() != null) { - errorMessage.append("mapped as aggregatable except in "); - errorMessage.append(Arrays.toString(fieldCap.nonAggregatableIndices())); - } - if (fieldCap.isSearchable() && fieldCap.nonSearchableIndices() != null) { - if (errorMessage.length() > 0) { - errorMessage.append(","); - } - errorMessage.append("mapped as searchable except in "); - errorMessage.append(Arrays.toString(fieldCap.nonSearchableIndices())); - } - - if (errorMessage.length() > 0) { - return new InvalidMappedField(fieldName, errorMessage.toString()); - } - } - - // everything checks - return null; - }; - - // merge all indices onto the same one - List indices = buildIndices( - typeRegistry, - null, - fieldCapsResponse, - null, - i -> indexPattern, - validityVerifier, - fieldUpdater, - allowedMetadataFields - ); - - if (indices.size() > 1) { - throw new QlIllegalArgumentException( - "Incorrect merging of mappings (likely due to a bug) - expect at most one but found [{}]", - indices.size() - ); - } - - String[] indexNames = fieldCapsResponse.getIndices(); - if (indices.isEmpty()) { - return IndexResolution.valid(new EsIndex(indexNames[0], emptyMap(), Set.of())); - } else { - EsIndex idx = indices.get(0); - return IndexResolution.valid(new EsIndex(idx.name(), idx.mapping(), Set.of(indexNames))); - } - } - - public static IndexResolution mergedMappings( - DataTypeRegistry typeRegistry, - String indexPattern, - FieldCapabilitiesResponse fieldCapsResponse - ) { - return mergedMappings(typeRegistry, indexPattern, fieldCapsResponse, (fieldName, types) -> null, null, null); - } - - private static EsField createField( - DataTypeRegistry typeRegistry, - String fieldName, - Map> globalCaps, - Map hierarchicalMapping, - Map flattedMapping, - Function field - ) { - - Map parentProps = hierarchicalMapping; - - int dot = fieldName.lastIndexOf('.'); - String fullFieldName = fieldName; - EsField parent = null; - - if (dot >= 0) { - String parentName = fieldName.substring(0, dot); - fieldName = fieldName.substring(dot + 1); - parent = flattedMapping.get(parentName); - if (parent == null) { - Map map = globalCaps.get(parentName); - Function fieldFunction; - - // lack of parent implies the field is an alias - if (map == null) { - // as such, create the field manually, marking the field to also be an alias - fieldFunction = s -> createField(typeRegistry, s, OBJECT.esType(), null, new TreeMap<>(), false, true); - } else { - Iterator iterator = map.values().iterator(); - FieldCapabilities parentCap = iterator.next(); - if (iterator.hasNext() && UNMAPPED.equals(parentCap.getType())) { - parentCap = iterator.next(); - } - final FieldCapabilities parentC = parentCap; - fieldFunction = s -> createField( - typeRegistry, - s, - parentC.getType(), - parentC.getMetricType(), - new TreeMap<>(), - parentC.isAggregatable(), - false - ); - } - - parent = createField(typeRegistry, parentName, globalCaps, hierarchicalMapping, flattedMapping, fieldFunction); - } - parentProps = parent.getProperties(); - } - - EsField esField = field.apply(fieldName); - - if (parent instanceof UnsupportedEsField unsupportedParent) { - String inherited = unsupportedParent.getInherited(); - String type = unsupportedParent.getOriginalType(); - - if (inherited == null) { - // mark the sub-field as unsupported, just like its parent, setting the first unsupported parent as the current one - esField = new UnsupportedEsField(esField.getName(), type, unsupportedParent.getName(), esField.getProperties()); - } else { - // mark the sub-field as unsupported, just like its parent, but setting the first unsupported parent - // as the parent's first unsupported grandparent - esField = new UnsupportedEsField(esField.getName(), type, inherited, esField.getProperties()); - } - } - - parentProps.put(fieldName, esField); - flattedMapping.put(fullFieldName, esField); - - return esField; - } - - private static EsField createField( - DataTypeRegistry typeRegistry, - String fieldName, - String typeName, - TimeSeriesParams.MetricType metricType, - Map props, - boolean isAggregateable, - boolean isAlias - ) { - DataType esType = typeRegistry.fromEs(typeName, metricType); - - if (esType == TEXT) { - return new TextEsField(fieldName, props, false, isAlias); - } - if (esType == KEYWORD) { - int length = Short.MAX_VALUE; - // TODO: to check whether isSearchable/isAggregateable takes into account the presence of the normalizer - boolean normalized = false; - return new KeywordEsField(fieldName, props, isAggregateable, length, normalized, isAlias); - } - if (esType == DATETIME) { - return DateEsField.dateEsField(fieldName, props, isAggregateable); - } - if (esType == UNSUPPORTED) { - String originalType = metricType == TimeSeriesParams.MetricType.COUNTER ? "counter" : typeName; - return new UnsupportedEsField(fieldName, originalType, null, props); - } - - return new EsField(fieldName, esType, props, isAggregateable, isAlias); - } - - private static FieldCapabilitiesRequest createFieldCapsRequest( - String index, - Set fieldNames, - IndicesOptions indicesOptions, - Map runtimeMappings - ) { - return new FieldCapabilitiesRequest().indices(Strings.commaDelimitedListToStringArray(index)) - .fields(fieldNames.toArray(String[]::new)) - .includeUnmapped(true) - .runtimeFields(runtimeMappings) - // lenient because we throw our own errors looking at the response e.g. if something was not resolved - // also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable - .indicesOptions(indicesOptions); - } - - private static FieldCapabilitiesRequest createFieldCapsRequest( - String index, - Set fieldNames, - boolean includeFrozen, - Map runtimeMappings - ) { - IndicesOptions indicesOptions = includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS; - return createFieldCapsRequest(index, fieldNames, indicesOptions, runtimeMappings); - } - - /** - * Resolves a pattern to multiple, separate indices. Doesn't perform validation. - */ - public void resolveAsSeparateMappings( - String indexWildcard, - String javaRegex, - boolean includeFrozen, - Map runtimeMappings, - ActionListener> listener - ) { - FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard, ALL_FIELDS, includeFrozen, runtimeMappings); - client.fieldCaps(fieldRequest, listener.delegateFailureAndWrap((delegate, response) -> { - client.admin().indices().getAliases(createGetAliasesRequest(response, includeFrozen), wrap(aliases -> { - delegate.onResponse(separateMappings(typeRegistry, javaRegex, response, aliases.getAliases())); - }, ex -> { - if (ex instanceof IndexNotFoundException || ex instanceof ElasticsearchSecurityException) { - delegate.onResponse(separateMappings(typeRegistry, javaRegex, response, null)); - } else { - delegate.onFailure(ex); - } - })); - })); - - } - - private static GetAliasesRequest createGetAliasesRequest(FieldCapabilitiesResponse response, boolean includeFrozen) { - return new GetAliasesRequest().aliases("*") - .indices(response.getIndices()) - .indicesOptions(includeFrozen ? FIELD_CAPS_FROZEN_INDICES_OPTIONS : FIELD_CAPS_INDICES_OPTIONS); - } - - public static List separateMappings( - DataTypeRegistry typeRegistry, - String javaRegex, - FieldCapabilitiesResponse fieldCaps, - Map> aliases - ) { - return buildIndices(typeRegistry, javaRegex, fieldCaps, aliases, Function.identity(), (s, cap) -> null, null, null); - } - - private static class Fields { - final Map hierarchicalMapping = new TreeMap<>(); - final Map flattedMapping = new LinkedHashMap<>(); - } - - /** - * Assemble an index-based mapping from the field caps (which is field based) by looking at the indices associated with - * each field. - */ - private static List buildIndices( - DataTypeRegistry typeRegistry, - String javaRegex, - FieldCapabilitiesResponse fieldCapsResponse, - Map> aliases, - Function indexNameProcessor, - BiFunction, InvalidMappedField> validityVerifier, - BiConsumer fieldUpdater, - Set allowedMetadataFields - ) { - - if ((fieldCapsResponse.getIndices() == null || fieldCapsResponse.getIndices().length == 0) - && (aliases == null || aliases.isEmpty())) { - return emptyList(); - } - - Set resolvedAliases = new HashSet<>(); - if (aliases != null) { - for (var aliasList : aliases.values()) { - for (AliasMetadata alias : aliasList) { - resolvedAliases.add(alias.getAlias()); - } - } - } - - Map indices = Maps.newLinkedHashMapWithExpectedSize(fieldCapsResponse.getIndices().length + resolvedAliases.size()); - Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null; - - // sort fields in reverse order to build the field hierarchy - TreeMap> sortedFields = new TreeMap<>(Collections.reverseOrder()); - final Map> fieldCaps = fieldCapsResponse.get(); - for (Entry> entry : fieldCaps.entrySet()) { - String fieldName = entry.getKey(); - // skip specific metadata fields - if ((allowedMetadataFields != null && allowedMetadataFields.contains(fieldName)) - || fieldCapsResponse.isMetadataField(fieldName) == false) { - sortedFields.put(fieldName, entry.getValue()); - } - } - - for (Entry> entry : sortedFields.entrySet()) { - String fieldName = entry.getKey(); - Map types = entry.getValue(); - final InvalidMappedField invalidField = validityVerifier.apply(fieldName, types); - // apply verification for fields belonging to index aliases - Map invalidFieldsForAliases = getInvalidFieldsForAliases(fieldName, types, aliases); - // For ESQL there are scenarios where there is no field asked from field_caps and the field_caps response only contains - // the list of indices. To be able to still have an "indices" list properly built (even if empty), the metadata fields are - // accepted but not actually added to each index hierarchy. - boolean isMetadataField = allowedMetadataFields != null && allowedMetadataFields.contains(fieldName); - - // check each type - for (Entry typeEntry : types.entrySet()) { - if (UNMAPPED.equals(typeEntry.getKey())) { - continue; - } - FieldCapabilities typeCap = typeEntry.getValue(); - String[] capIndices = typeCap.indices(); - - // compute the actual indices - if any are specified, take into account the unmapped indices - final String[] concreteIndices; - if (capIndices != null) { - concreteIndices = capIndices; - } else { - concreteIndices = fieldCapsResponse.getIndices(); - } - - Set uniqueAliases = new LinkedHashSet<>(); - // put the field in their respective mappings and collect the aliases names - for (String index : concreteIndices) { - List concreteIndexAliases = aliases != null ? aliases.get(index) : null; - if (concreteIndexAliases != null) { - for (AliasMetadata e : concreteIndexAliases) { - uniqueAliases.add(e.alias()); - } - } - // TODO is split still needed? - if (pattern == null || pattern.matcher(splitQualifiedIndex(index).v2()).matches()) { - String indexName = indexNameProcessor.apply(index); - Fields indexFields = indices.computeIfAbsent(indexName, k -> new Fields()); - EsField field = indexFields.flattedMapping.get(fieldName); - // create field hierarchy or update it in case of an invalid field - if (isMetadataField == false - && (field == null || (invalidField != null && (field instanceof InvalidMappedField) == false))) { - createField(typeRegistry, fieldName, indexFields, fieldCaps, invalidField, typeCap); - - // In evolving mappings, it is possible for a field to be promoted to an object in new indices - // meaning there are subfields associated with this *invalid* field. - // index_A: file -> keyword - // index_B: file -> object, file.name = keyword - // - // In the scenario above file is problematic but file.name is not. This scenario is addressed - // below through the dedicated callback - copy the existing properties or drop them all together. - // Note this applies for *invalid* fields (that have conflicts), not *unsupported* (those that cannot be read) - // See https://github.com/elastic/elasticsearch/pull/100875 - - // Postpone the call until is really needed - if (fieldUpdater != null && field != null) { - EsField newField = indexFields.flattedMapping.get(fieldName); - if (newField != field && newField instanceof InvalidMappedField newInvalidField) { - fieldUpdater.accept(field, newInvalidField); - } - } - } - } - } - // put the field in their respective mappings by alias name - for (String index : uniqueAliases) { - Fields indexFields = indices.computeIfAbsent(index, k -> new Fields()); - EsField field = indexFields.flattedMapping.get(fieldName); - if (isMetadataField == false && field == null && invalidFieldsForAliases.get(index) == null) { - createField(typeRegistry, fieldName, indexFields, fieldCaps, invalidField, typeCap); - } - } - } - } - - // return indices in ascending order - List foundIndices = new ArrayList<>(indices.size()); - for (Entry entry : indices.entrySet()) { - foundIndices.add(new EsIndex(entry.getKey(), entry.getValue().hierarchicalMapping, Set.of(entry.getKey()))); - } - foundIndices.sort(Comparator.comparing(EsIndex::name)); - return foundIndices; - } - - private static void createField( - DataTypeRegistry typeRegistry, - String fieldName, - Fields indexFields, - Map> fieldCaps, - InvalidMappedField invalidField, - FieldCapabilities typeCap - ) { - int dot = fieldName.lastIndexOf('.'); - /* - * Looking up the "tree" at the parent fields here to see if the field is an alias. - * When the upper elements of the "tree" have no elements in fieldcaps, then this is an alias field. But not - * always: if there are two aliases - a.b.c.alias1 and a.b.c.alias2 - only one of them will be considered alias. - */ - Holder isAliasFieldType = new Holder<>(false); - if (dot >= 0) { - String parentName = fieldName.substring(0, dot); - if (indexFields.flattedMapping.get(parentName) == null) { - // lack of parent implies the field is an alias - if (fieldCaps.get(parentName) == null) { - isAliasFieldType.set(true); - } - } - } - - createField( - typeRegistry, - fieldName, - fieldCaps, - indexFields.hierarchicalMapping, - indexFields.flattedMapping, - s -> invalidField != null - ? invalidField - : createField( - typeRegistry, - s, - typeCap.getType(), - typeCap.getMetricType(), - new TreeMap<>(), - typeCap.isAggregatable(), - isAliasFieldType.get() - ) - ); - } - - /* - * Checks if the field is valid (same type and same capabilities - searchable/aggregatable) across indices belonging to a list - * of aliases. - * A field can look like the example below (generated by field_caps API). - * "name": { - * "text": { - * "type": "text", - * "searchable": false, - * "aggregatable": false, - * "indices": [ - * "bar", - * "foo" - * ], - * "non_searchable_indices": [ - * "foo" - * ] - * }, - * "keyword": { - * "type": "keyword", - * "searchable": false, - * "aggregatable": true, - * "non_aggregatable_indices": [ - * "bar", "baz" - * ] - * } - * } - */ - private static Map getInvalidFieldsForAliases( - String fieldName, - Map types, - Map> aliases - ) { - if (aliases == null || aliases.isEmpty()) { - return emptyMap(); - } - Map invalidFields = new HashMap<>(); - Map> typesErrors = new HashMap<>(); // map holding aliases and a list of unique field types across its indices - Map> aliasToIndices = new HashMap<>(); // map with aliases and their list of indices - - for (var entry : aliases.entrySet()) { - for (AliasMetadata aliasMetadata : entry.getValue()) { - String aliasName = aliasMetadata.alias(); - aliasToIndices.putIfAbsent(aliasName, new HashSet<>()); - aliasToIndices.get(aliasName).add(entry.getKey()); - } - } - - // iterate over each type - for (Entry type : types.entrySet()) { - String esFieldType = type.getKey(); - if (Objects.equals(esFieldType, UNMAPPED)) { - continue; - } - String[] indices = type.getValue().indices(); - // if there is a list of indices where this field type is defined - if (indices != null) { - // Look at all these indices' aliases and add the type of the field to a list (Set) with unique elements. - // A valid mapping for a field in an index alias should contain only one type. If it doesn't, this means that field - // is mapped as different types across the indices in this index alias. - for (String index : indices) { - List indexAliases = aliases.get(index); - if (indexAliases == null) { - continue; - } - for (AliasMetadata aliasMetadata : indexAliases) { - String aliasName = aliasMetadata.alias(); - if (typesErrors.containsKey(aliasName)) { - typesErrors.get(aliasName).add(esFieldType); - } else { - Set fieldTypes = new HashSet<>(); - fieldTypes.add(esFieldType); - typesErrors.put(aliasName, fieldTypes); - } - } - } - } - } - - for (String aliasName : aliasToIndices.keySet()) { - // if, for the same index alias, there are multiple field types for this fieldName ie the index alias has indices where the same - // field name is of different types - Set esFieldTypes = typesErrors.get(aliasName); - if (esFieldTypes != null && esFieldTypes.size() > 1) { - // consider the field as invalid, for the currently checked index alias - // the error message doesn't actually matter - invalidFields.put(aliasName, new InvalidMappedField(fieldName)); - } else { - // if the field type is the same across all this alias' indices, check the field's capabilities (searchable/aggregatable) - for (Entry type : types.entrySet()) { - if (Objects.equals(type.getKey(), UNMAPPED)) { - continue; - } - FieldCapabilities f = type.getValue(); - - // the existence of a list of non_aggregatable_indices is an indication that not all indices have the same capabilities - // but this list can contain indices belonging to other aliases, so we need to check only for this alias - if (f.nonAggregatableIndices() != null) { - Set aliasIndices = aliasToIndices.get(aliasName); - int nonAggregatableCount = 0; - // either all or none of the non-aggregatable indices belonging to a certain alias should be in this list - for (String nonAggIndex : f.nonAggregatableIndices()) { - if (aliasIndices.contains(nonAggIndex)) { - nonAggregatableCount++; - } - } - if (nonAggregatableCount > 0 && nonAggregatableCount != aliasIndices.size()) { - invalidFields.put(aliasName, new InvalidMappedField(fieldName)); - break; - } - } - - // perform the same check for non_searchable_indices list - if (f.nonSearchableIndices() != null) { - Set aliasIndices = aliasToIndices.get(aliasName); - int nonSearchableCount = 0; - // either all or none of the non-searchable indices belonging to a certain alias should be in this list - for (String nonSearchIndex : f.nonSearchableIndices()) { - if (aliasIndices.contains(nonSearchIndex)) { - nonSearchableCount++; - } - } - if (nonSearchableCount > 0 && nonSearchableCount != aliasIndices.size()) { - invalidFields.put(aliasName, new InvalidMappedField(fieldName)); - break; - } - } - } - } - } - - if (invalidFields.size() > 0) { - return invalidFields; - } - // everything checks - return emptyMap(); - } - - /** - * Callback interface used when transitioning an already discovered EsField to an InvalidMapped one. - * By default, this interface is not used, meaning when a field is marked as invalid all its subfields - * are removed (are dropped). - * For cases where this is not desired, a different strategy can be employed such as keeping the properties: - * @see IndexResolver#PRESERVE_PROPERTIES - */ - public interface ExistingFieldInvalidCallback extends BiConsumer {}; - - /** - * Preserve the properties (sub fields) of an existing field even when marking it as invalid. - */ - public static ExistingFieldInvalidCallback PRESERVE_PROPERTIES = (oldField, newField) -> { - var oldProps = oldField.getProperties(); - if (oldProps.size() > 0) { - newField.getProperties().putAll(oldProps); - } - }; -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/RemoteClusterResolver.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/RemoteClusterResolver.java deleted file mode 100644 index e83eddc71000b..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/RemoteClusterResolver.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.core.index; - -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.transport.RemoteClusterAware; -import org.elasticsearch.transport.RemoteConnectionStrategy; - -import java.util.Set; -import java.util.TreeSet; -import java.util.concurrent.CopyOnWriteArraySet; - -public final class RemoteClusterResolver extends RemoteClusterAware { - private final CopyOnWriteArraySet clusters; - - public RemoteClusterResolver(Settings settings, ClusterSettings clusterSettings) { - super(settings); - clusters = new CopyOnWriteArraySet<>(getEnabledRemoteClusters(settings)); - listenForUpdates(clusterSettings); - } - - @Override - protected void updateRemoteCluster(String clusterAlias, Settings settings) { - if (RemoteConnectionStrategy.isConnectionEnabled(clusterAlias, settings)) { - clusters.add(clusterAlias); - } else { - clusters.remove(clusterAlias); - } - } - - public Set remoteClusters() { - return new TreeSet<>(clusters); - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/VersionCompatibilityChecks.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/VersionCompatibilityChecks.java deleted file mode 100644 index e4ae4f8f0d51f..0000000000000 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/index/VersionCompatibilityChecks.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.esql.core.index; - -import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; -import org.elasticsearch.Version; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.xpack.esql.core.type.DataType; - -import static org.elasticsearch.Version.V_8_2_0; -import static org.elasticsearch.Version.V_8_4_0; -import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; -import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; - -public final class VersionCompatibilityChecks { - - public static final Version INTRODUCING_UNSIGNED_LONG = V_8_2_0; - public static final TransportVersion INTRODUCING_UNSIGNED_LONG_TRANSPORT = TransportVersions.V_8_2_0; - public static final Version INTRODUCING_VERSION_FIELD_TYPE = V_8_4_0; - - private VersionCompatibilityChecks() {} - - public static boolean isTypeSupportedInVersion(DataType dataType, Version version) { - if (dataType == UNSIGNED_LONG) { - return supportsUnsignedLong(version); - } - if (dataType == VERSION) { - return supportsVersionType(version); - } - return true; - } - - /** - * Does the provided {@code version} support the unsigned_long type (PR#60050)? - */ - public static boolean supportsUnsignedLong(Version version) { - return INTRODUCING_UNSIGNED_LONG.compareTo(version) <= 0; - } - - /** - * Does the provided {@code version} support the version type (PR#85502)? - */ - public static boolean supportsVersionType(Version version) { - return INTRODUCING_VERSION_FIELD_TYPE.compareTo(version) <= 0; - } - - public static @Nullable Version versionIntroducingType(DataType dataType) { - if (dataType == UNSIGNED_LONG) { - return INTRODUCING_UNSIGNED_LONG; - } - if (dataType == VERSION) { - return INTRODUCING_VERSION_FIELD_TYPE; - } - - return null; - } -} diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 9d6a325a6028f..7f48751535ba9 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.index.mapper.SourceFieldMapper; @@ -18,6 +19,8 @@ import java.util.Comparator; import java.util.Locale; import java.util.Map; +import java.util.Set; +import java.util.function.Function; import java.util.stream.Stream; import static java.util.stream.Collectors.toMap; @@ -144,6 +147,15 @@ public enum DataType { ES_TO_TYPE = Collections.unmodifiableMap(map); } + private static final Map NAME_OR_ALIAS_TO_TYPE; + static { + Map map = DataType.types().stream().collect(toMap(DataType::typeName, Function.identity())); + map.put("bool", BOOLEAN); + map.put("int", INTEGER); + map.put("string", KEYWORD); + NAME_OR_ALIAS_TO_TYPE = Collections.unmodifiableMap(map); + } + public static Collection types() { return TYPES; } @@ -188,7 +200,7 @@ public static DataType fromJava(Object value) { if (value instanceof ZonedDateTime) { return DATETIME; } - if (value instanceof String || value instanceof Character) { + if (value instanceof String || value instanceof Character || value instanceof BytesRef) { return KEYWORD; } @@ -282,4 +294,13 @@ public static DataType readFrom(StreamInput in) throws IOException { } return dataType; } + + public static Set namesAndAliases() { + return NAME_OR_ALIAS_TO_TYPE.keySet(); + } + + public static DataType fromNameOrAlias(String typeName) { + DataType type = NAME_OR_ALIAS_TO_TYPE.get(typeName.toLowerCase(Locale.ROOT)); + return type != null ? type : UNSUPPORTED; + } } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java index fd7bfbec4730f..9b088cfb19f6c 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java @@ -15,11 +15,15 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.TreeMap; /** * Representation of field mapped differently across indices. * Used during mapping discovery only. + * Note that the field typesToIndices is not serialized because that information is + * not required through the cluster, only surviving as long as the Analyser phase of query planning. + * It is used specifically for the 'union types' feature in ES|QL. */ public class InvalidMappedField extends EsField { static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -29,10 +33,10 @@ public class InvalidMappedField extends EsField { ); private final String errorMessage; + private final Map> typesToIndices; public InvalidMappedField(String name, String errorMessage, Map properties) { - super(name, DataType.UNSUPPORTED, properties, false); - this.errorMessage = errorMessage; + this(name, errorMessage, properties, Map.of()); } public InvalidMappedField(String name, String errorMessage) { @@ -43,6 +47,19 @@ public InvalidMappedField(String name) { this(name, StringUtils.EMPTY, new TreeMap<>()); } + /** + * Constructor supporting union types, used in ES|QL. + */ + public InvalidMappedField(String name, Map> typesToIndices) { + this(name, makeErrorMessage(typesToIndices), new TreeMap<>(), typesToIndices); + } + + private InvalidMappedField(String name, String errorMessage, Map properties, Map> typesToIndices) { + super(name, DataType.UNSUPPORTED, properties, false); + this.errorMessage = errorMessage; + this.typesToIndices = typesToIndices; + } + private InvalidMappedField(StreamInput in) throws IOException { this(in.readString(), in.readString(), in.readImmutableMap(StreamInput::readString, i -> i.readNamedWriteable(EsField.class))); } @@ -88,4 +105,28 @@ public EsField getExactField() { public Exact getExactInfo() { return new Exact(false, "Field [" + getName() + "] is invalid, cannot access it"); } + + public Map> getTypesToIndices() { + return typesToIndices; + } + + private static String makeErrorMessage(Map> typesToIndices) { + StringBuilder errorMessage = new StringBuilder(); + errorMessage.append("mapped as ["); + errorMessage.append(typesToIndices.size()); + errorMessage.append("] incompatible types: "); + boolean first = true; + for (Map.Entry> e : typesToIndices.entrySet()) { + if (first) { + first = false; + } else { + errorMessage.append(", "); + } + errorMessage.append("["); + errorMessage.append(e.getKey()); + errorMessage.append("] in "); + errorMessage.append(e.getValue()); + } + return errorMessage.toString(); + } } diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/LiteralTests.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/LiteralTests.java index 7e57e8f358ae1..a4c67a8076479 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/LiteralTests.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/LiteralTests.java @@ -76,6 +76,10 @@ protected Literal copy(Literal instance) { @Override protected Literal mutate(Literal instance) { + return mutateLiteral(instance); + } + + public static Literal mutateLiteral(Literal instance) { List> mutators = new ArrayList<>(); // Changing the location doesn't count as mutation because..... it just doesn't, ok?! // Change the value to another valid value @@ -116,7 +120,7 @@ public void testReplaceChildren() { assertEquals("this type of node doesn't have any children to replace", e.getMessage()); } - private Object randomValueOfTypeOtherThan(Object original, DataType type) { + private static Object randomValueOfTypeOtherThan(Object original, DataType type) { for (ValueAndCompatibleTypes gen : GENERATORS) { if (gen.validDataTypes.get(0) == type) { return randomValueOtherThan(original, () -> DataTypeConverter.convert(gen.valueSupplier.get(), type)); @@ -125,7 +129,7 @@ private Object randomValueOfTypeOtherThan(Object original, DataType type) { throw new IllegalArgumentException("No native generator for [" + type + "]"); } - private List validReplacementDataTypes(Object value, DataType type) { + private static List validReplacementDataTypes(Object value, DataType type) { List validDataTypes = new ArrayList<>(); List options = Arrays.asList(BYTE, SHORT, INTEGER, LONG, FLOAT, DOUBLE, BOOLEAN); for (DataType candidate : options) { diff --git a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/FunctionTestUtils.java b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/FunctionTestUtils.java index 8f0ff30074b83..9e3ab40ec6462 100644 --- a/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/FunctionTestUtils.java +++ b/x-pack/plugin/esql-core/src/test/java/org/elasticsearch/xpack/esql/core/expression/function/scalar/FunctionTestUtils.java @@ -7,20 +7,10 @@ package org.elasticsearch.xpack.esql.core.expression.function.scalar; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.type.DataType; -import java.time.Instant; -import java.time.ZonedDateTime; -import java.util.BitSet; -import java.util.Iterator; - import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; -import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; -import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; -import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; -import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; public final class FunctionTestUtils { @@ -31,61 +21,4 @@ public static Literal l(Object value) { public static Literal l(Object value, DataType type) { return new Literal(EMPTY, value, type); } - - public static Literal randomStringLiteral() { - return l(ESTestCase.randomRealisticUnicodeOfLength(10), KEYWORD); - } - - public static Literal randomIntLiteral() { - return l(ESTestCase.randomInt(), INTEGER); - } - - public static Literal randomBooleanLiteral() { - return l(ESTestCase.randomBoolean(), BOOLEAN); - } - - public static Literal randomDatetimeLiteral() { - return l(ZonedDateTime.ofInstant(Instant.ofEpochMilli(ESTestCase.randomLong()), ESTestCase.randomZone()), DATETIME); - } - - public static class Combinations implements Iterable { - private int n; - private int k; - - public Combinations(int n, int k) { - this.n = n; - this.k = k; - } - - @Override - public Iterator iterator() { - return new Iterator<>() { - BitSet bs = new BitSet(n); - - { - bs.set(0, k); - } - - @Override - public boolean hasNext() { - return bs != null; - } - - @Override - public BitSet next() { - BitSet old = (BitSet) bs.clone(); - int b = bs.previousClearBit(n - 1); - int b1 = bs.previousSetBit(b); - if (b1 == -1) { - bs = null; - } else { - bs.clear(b1); - bs.set(b1 + 1, b1 + (n - b) + 1); - bs.clear(b1 + (n - b) + 1, n); - } - return old; - } - }; - } - } } diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 92071543aa27e..dbec0963d1aab 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -28,7 +28,9 @@ dependencies { // Also contains a dummy processor to allow compilation with unused annotations. annotationProcessor project('compute:gen') - testImplementation project('qa:testFixtures') + testImplementation(project('qa:testFixtures')) { + exclude(group:"org.elasticsearch.plugin", module: "esql") + } testImplementation project(':test:framework') testImplementation(testArtifact(project(xpackModule('core')))) testImplementation project(path: xpackModule('enrich')) diff --git a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java index 7e92fc5c2734e..0216ea07e5c7c 100644 --- a/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java +++ b/x-pack/plugin/esql/compute/ann/src/main/java/org/elasticsearch/compute/ann/GroupingAggregator.java @@ -12,6 +12,10 @@ import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +/** + * Annotates a class that implements an aggregation function with grouping. + * See {@link Aggregator} for more information. + */ @Target(ElementType.TYPE) @Retention(RetentionPolicy.SOURCE) public @interface GroupingAggregator { diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle index a9b16d19aecb8..c7fa29c6a91f0 100644 --- a/x-pack/plugin/esql/compute/build.gradle +++ b/x-pack/plugin/esql/compute/build.gradle @@ -36,16 +36,18 @@ spotless { } } -def prop(Type, type, TYPE, BYTES, Array, Hash) { +def prop(Type, type, Wrapper, TYPE, BYTES, Array, Hash) { return [ "Type" : Type, "type" : type, + "Wrapper": Wrapper, "TYPE" : TYPE, "BYTES" : BYTES, "Array" : Array, "Hash" : Hash, "int" : type == "int" ? "true" : "", + "float" : type == "float" ? "true" : "", "long" : type == "long" ? "true" : "", "double" : type == "double" ? "true" : "", "BytesRef" : type == "BytesRef" ? "true" : "", @@ -54,11 +56,13 @@ def prop(Type, type, TYPE, BYTES, Array, Hash) { } tasks.named('stringTemplates').configure { - var intProperties = prop("Int", "int", "INT", "Integer.BYTES", "IntArray", "LongHash") - var longProperties = prop("Long", "long", "LONG", "Long.BYTES", "LongArray", "LongHash") - var doubleProperties = prop("Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray", "LongHash") - var bytesRefProperties = prop("BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "", "BytesRefHash") - var booleanProperties = prop("Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray", "") + var intProperties = prop("Int", "int", "Integer", "INT", "Integer.BYTES", "IntArray", "LongHash") + var floatProperties = prop("Float", "float", "Float", "FLOAT", "Float.BYTES", "FloatArray", "LongHash") + var longProperties = prop("Long", "long", "Long", "LONG", "Long.BYTES", "LongArray", "LongHash") + var doubleProperties = prop("Double", "double", "Double", "DOUBLE", "Double.BYTES", "DoubleArray", "LongHash") + var bytesRefProperties = prop("BytesRef", "BytesRef", "", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "", "BytesRefHash") + var booleanProperties = prop("Boolean", "boolean", "Boolean", "BOOLEAN", "Byte.BYTES", "BitArray", "") + // primitive vectors File vectorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st") template { @@ -66,6 +70,11 @@ tasks.named('stringTemplates').configure { it.inputFile = vectorInputFile it.outputFile = "org/elasticsearch/compute/data/IntVector.java" } + template { + it.properties = floatProperties + it.inputFile = vectorInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatVector.java" + } template { it.properties = longProperties it.inputFile = vectorInputFile @@ -93,6 +102,11 @@ tasks.named('stringTemplates').configure { it.inputFile = arrayVectorInputFile it.outputFile = "org/elasticsearch/compute/data/IntArrayVector.java" } + template { + it.properties = floatProperties + it.inputFile = arrayVectorInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatArrayVector.java" + } template { it.properties = longProperties it.inputFile = arrayVectorInputFile @@ -120,6 +134,11 @@ tasks.named('stringTemplates').configure { it.inputFile = bigArrayVectorInputFile it.outputFile = "org/elasticsearch/compute/data/IntBigArrayVector.java" } + template { + it.properties = floatProperties + it.inputFile = bigArrayVectorInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatBigArrayVector.java" + } template { it.properties = longProperties it.inputFile = bigArrayVectorInputFile @@ -142,6 +161,11 @@ tasks.named('stringTemplates').configure { it.inputFile = constantVectorInputFile it.outputFile = "org/elasticsearch/compute/data/ConstantIntVector.java" } + template { + it.properties = floatProperties + it.inputFile = constantVectorInputFile + it.outputFile = "org/elasticsearch/compute/data/ConstantFloatVector.java" + } template { it.properties = longProperties it.inputFile = constantVectorInputFile @@ -169,6 +193,11 @@ tasks.named('stringTemplates').configure { it.inputFile = blockInputFile it.outputFile = "org/elasticsearch/compute/data/IntBlock.java" } + template { + it.properties = floatProperties + it.inputFile = blockInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatBlock.java" + } template { it.properties = longProperties it.inputFile = blockInputFile @@ -196,6 +225,11 @@ tasks.named('stringTemplates').configure { it.inputFile = arrayBlockInputFile it.outputFile = "org/elasticsearch/compute/data/IntArrayBlock.java" } + template { + it.properties = floatProperties + it.inputFile = arrayBlockInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatArrayBlock.java" + } template { it.properties = longProperties it.inputFile = arrayBlockInputFile @@ -223,6 +257,11 @@ tasks.named('stringTemplates').configure { it.inputFile = bigArrayBlockInputFile it.outputFile = "org/elasticsearch/compute/data/IntBigArrayBlock.java" } + template { + it.properties = floatProperties + it.inputFile = bigArrayBlockInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatBigArrayBlock.java" + } template { it.properties = longProperties it.inputFile = bigArrayBlockInputFile @@ -245,6 +284,11 @@ tasks.named('stringTemplates').configure { it.inputFile = vectorBlockInputFile it.outputFile = "org/elasticsearch/compute/data/IntVectorBlock.java" } + template { + it.properties = floatProperties + it.inputFile = vectorBlockInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatVectorBlock.java" + } template { it.properties = longProperties it.inputFile = vectorBlockInputFile @@ -272,6 +316,11 @@ tasks.named('stringTemplates').configure { it.inputFile = blockBuildersInputFile it.outputFile = "org/elasticsearch/compute/data/IntBlockBuilder.java" } + template { + it.properties = floatProperties + it.inputFile = blockBuildersInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatBlockBuilder.java" + } template { it.properties = longProperties it.inputFile = blockBuildersInputFile @@ -299,6 +348,11 @@ tasks.named('stringTemplates').configure { it.inputFile = vectorBuildersInputFile it.outputFile = "org/elasticsearch/compute/data/IntVectorBuilder.java" } + template { + it.properties = floatProperties + it.inputFile = vectorBuildersInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatVectorBuilder.java" + } template { it.properties = longProperties it.inputFile = vectorBuildersInputFile @@ -325,6 +379,11 @@ tasks.named('stringTemplates').configure { it.inputFile = vectorFixedBuildersInputFile it.outputFile = "org/elasticsearch/compute/data/IntVectorFixedBuilder.java" } + template { + it.properties = floatProperties + it.inputFile = vectorFixedBuildersInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatVectorFixedBuilder.java" + } template { it.properties = longProperties it.inputFile = vectorFixedBuildersInputFile @@ -351,12 +410,17 @@ tasks.named('stringTemplates').configure { it.inputFile = stateInputFile it.outputFile = "org/elasticsearch/compute/aggregation/LongState.java" } + template { + it.properties = floatProperties + it.inputFile = stateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/FloatState.java" + } template { it.properties = doubleProperties it.inputFile = stateInputFile it.outputFile = "org/elasticsearch/compute/aggregation/DoubleState.java" } - // block builders + // block lookups File lookupInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/X-Lookup.java.st") template { it.properties = intProperties @@ -368,6 +432,11 @@ tasks.named('stringTemplates').configure { it.inputFile = lookupInputFile it.outputFile = "org/elasticsearch/compute/data/LongLookup.java" } + template { + it.properties = floatProperties + it.inputFile = lookupInputFile + it.outputFile = "org/elasticsearch/compute/data/FloatLookup.java" + } template { it.properties = doubleProperties it.inputFile = lookupInputFile @@ -399,6 +468,11 @@ tasks.named('stringTemplates').configure { it.inputFile = arrayStateInputFile it.outputFile = "org/elasticsearch/compute/aggregation/DoubleArrayState.java" } + template { + it.properties = floatProperties + it.inputFile = arrayStateInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/FloatArrayState.java" + } File valuesAggregatorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st") template { it.properties = intProperties @@ -410,6 +484,11 @@ tasks.named('stringTemplates').configure { it.inputFile = valuesAggregatorInputFile it.outputFile = "org/elasticsearch/compute/aggregation/ValuesLongAggregator.java" } + template { + it.properties = floatProperties + it.inputFile = valuesAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java" + } template { it.properties = doubleProperties it.inputFile = valuesAggregatorInputFile @@ -432,12 +511,40 @@ tasks.named('stringTemplates').configure { it.inputFile = rateAggregatorInputFile it.outputFile = "org/elasticsearch/compute/aggregation/RateLongAggregator.java" } + template { + it.properties = floatProperties + it.inputFile = rateAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/RateFloatAggregator.java" + } template { it.properties = doubleProperties it.inputFile = rateAggregatorInputFile it.outputFile = "org/elasticsearch/compute/aggregation/RateDoubleAggregator.java" } + + File topListAggregatorInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/aggregation/X-TopListAggregator.java.st") + template { + it.properties = intProperties + it.inputFile = topListAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/TopListIntAggregator.java" + } + template { + it.properties = longProperties + it.inputFile = topListAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/TopListLongAggregator.java" + } + template { + it.properties = floatProperties + it.inputFile = topListAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/TopListFloatAggregator.java" + } + template { + it.properties = doubleProperties + it.inputFile = topListAggregatorInputFile + it.outputFile = "org/elasticsearch/compute/aggregation/TopListDoubleAggregator.java" + } + File multivalueDedupeInputFile = file("src/main/java/org/elasticsearch/compute/operator/mvdedupe/X-MultivalueDedupe.java.st") template { it.properties = intProperties @@ -501,6 +608,11 @@ tasks.named('stringTemplates').configure { it.inputFile = keyExtractorInputFile it.outputFile = "org/elasticsearch/compute/operator/topn/KeyExtractorForLong.java" } + template { + it.properties = floatProperties + it.inputFile = keyExtractorInputFile + it.outputFile = "org/elasticsearch/compute/operator/topn/KeyExtractorForFloat.java" + } template { it.properties = doubleProperties it.inputFile = keyExtractorInputFile @@ -527,6 +639,11 @@ tasks.named('stringTemplates').configure { it.inputFile = valueExtractorInputFile it.outputFile = "org/elasticsearch/compute/operator/topn/ValueExtractorForLong.java" } + template { + it.properties = floatProperties + it.inputFile = valueExtractorInputFile + it.outputFile = "org/elasticsearch/compute/operator/topn/ValueExtractorForFloat.java" + } template { it.properties = doubleProperties it.inputFile = valueExtractorInputFile @@ -558,4 +675,31 @@ tasks.named('stringTemplates').configure { it.inputFile = resultBuilderInputFile it.outputFile = "org/elasticsearch/compute/operator/topn/ResultBuilderForDouble.java" } + template { + it.properties = floatProperties + it.inputFile = resultBuilderInputFile + it.outputFile = "org/elasticsearch/compute/operator/topn/ResultBuilderForFloat.java" + } + + File bucketedSortInputFile = new File("${projectDir}/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st") + template { + it.properties = intProperties + it.inputFile = bucketedSortInputFile + it.outputFile = "org/elasticsearch/compute/data/sort/IntBucketedSort.java" + } + template { + it.properties = longProperties + it.inputFile = bucketedSortInputFile + it.outputFile = "org/elasticsearch/compute/data/sort/LongBucketedSort.java" + } + template { + it.properties = floatProperties + it.inputFile = bucketedSortInputFile + it.outputFile = "org/elasticsearch/compute/data/sort/FloatBucketedSort.java" + } + template { + it.properties = doubleProperties + it.inputFile = bucketedSortInputFile + it.outputFile = "org/elasticsearch/compute/data/sort/DoubleBucketedSort.java" + } } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java index d3fe51b4cc225..1127d4b4ccb72 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/AggregatorImplementer.java @@ -44,6 +44,8 @@ import static org.elasticsearch.compute.gen.Types.DOUBLE_VECTOR; import static org.elasticsearch.compute.gen.Types.DRIVER_CONTEXT; import static org.elasticsearch.compute.gen.Types.ELEMENT_TYPE; +import static org.elasticsearch.compute.gen.Types.FLOAT_BLOCK; +import static org.elasticsearch.compute.gen.Types.FLOAT_VECTOR; import static org.elasticsearch.compute.gen.Types.INTERMEDIATE_STATE_DESC; import static org.elasticsearch.compute.gen.Types.INT_BLOCK; import static org.elasticsearch.compute.gen.Types.INT_VECTOR; @@ -136,6 +138,8 @@ static String valueType(ExecutableElement init, ExecutableElement combine) { switch (initReturn) { case "double": return "double"; + case "float": + return "float"; case "long": return "long"; case "int": @@ -151,6 +155,7 @@ static ClassName valueBlockType(ExecutableElement init, ExecutableElement combin return switch (valueType(init, combine)) { case "boolean" -> BOOLEAN_BLOCK; case "double" -> DOUBLE_BLOCK; + case "float" -> FLOAT_BLOCK; case "long" -> LONG_BLOCK; case "int" -> INT_BLOCK; case "org.apache.lucene.util.BytesRef" -> BYTES_REF_BLOCK; @@ -162,6 +167,7 @@ static ClassName valueVectorType(ExecutableElement init, ExecutableElement combi return switch (valueType(init, combine)) { case "boolean" -> BOOLEAN_VECTOR; case "double" -> DOUBLE_VECTOR; + case "float" -> FLOAT_VECTOR; case "long" -> LONG_VECTOR; case "int" -> INT_VECTOR; case "org.apache.lucene.util.BytesRef" -> BYTES_REF_VECTOR; @@ -445,6 +451,8 @@ private String primitiveStateMethod() { return "longValue"; case "org.elasticsearch.compute.aggregation.DoubleState": return "doubleValue"; + case "org.elasticsearch.compute.aggregation.FloatState": + return "floatValue"; default: throw new IllegalArgumentException( "don't know how to fetch primitive values from " + stateType + ". define combineIntermediate." @@ -495,6 +503,9 @@ private void primitiveStateToResult(MethodSpec.Builder builder) { case "org.elasticsearch.compute.aggregation.DoubleState": builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantDoubleBlockWith(state.doubleValue(), 1)"); return; + case "org.elasticsearch.compute.aggregation.FloatState": + builder.addStatement("blocks[offset] = driverContext.blockFactory().newConstantFloatBlockWith(state.floatValue(), 1)"); + return; default: throw new IllegalArgumentException("don't know how to convert state to result: " + stateType); } @@ -521,7 +532,7 @@ private MethodSpec close() { private boolean hasPrimitiveState() { return switch (stateType.toString()) { case "org.elasticsearch.compute.aggregation.IntState", "org.elasticsearch.compute.aggregation.LongState", - "org.elasticsearch.compute.aggregation.DoubleState" -> true; + "org.elasticsearch.compute.aggregation.DoubleState", "org.elasticsearch.compute.aggregation.FloatState" -> true; default -> false; }; } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java index cb65d2337d588..c9cdcfe42fddd 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/GroupingAggregatorImplementer.java @@ -585,7 +585,7 @@ private MethodSpec close() { private boolean hasPrimitiveState() { return switch (stateType.toString()) { case "org.elasticsearch.compute.aggregation.IntArrayState", "org.elasticsearch.compute.aggregation.LongArrayState", - "org.elasticsearch.compute.aggregation.DoubleArrayState" -> true; + "org.elasticsearch.compute.aggregation.DoubleArrayState", "org.elasticsearch.compute.aggregation.FloatArrayState" -> true; default -> false; }; } diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java index 741a1294e6fb5..6f98f1f797ab0 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Methods.java @@ -211,6 +211,7 @@ static String vectorAccessorName(String elementTypeName) { case "INT" -> "getInt"; case "LONG" -> "getLong"; case "DOUBLE" -> "getDouble"; + case "FLOAT" -> "getFloat"; case "BYTES_REF" -> "getBytesRef"; default -> throw new IllegalArgumentException( "don't know how to fetch primitive values from " + elementTypeName + ". define combineIntermediate." diff --git a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java index 6618d9e4f41b5..3150741ddcb05 100644 --- a/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java +++ b/x-pack/plugin/esql/compute/gen/src/main/java/org/elasticsearch/compute/gen/Types.java @@ -41,12 +41,14 @@ public class Types { static final ClassName INT_BLOCK = ClassName.get(DATA_PACKAGE, "IntBlock"); static final ClassName LONG_BLOCK = ClassName.get(DATA_PACKAGE, "LongBlock"); static final ClassName DOUBLE_BLOCK = ClassName.get(DATA_PACKAGE, "DoubleBlock"); + static final ClassName FLOAT_BLOCK = ClassName.get(DATA_PACKAGE, "FloatBlock"); static final ClassName BOOLEAN_BLOCK_BUILDER = BOOLEAN_BLOCK.nestedClass("Builder"); static final ClassName BYTES_REF_BLOCK_BUILDER = BYTES_REF_BLOCK.nestedClass("Builder"); static final ClassName INT_BLOCK_BUILDER = INT_BLOCK.nestedClass("Builder"); static final ClassName LONG_BLOCK_BUILDER = LONG_BLOCK.nestedClass("Builder"); static final ClassName DOUBLE_BLOCK_BUILDER = DOUBLE_BLOCK.nestedClass("Builder"); + static final ClassName FLOAT_BLOCK_BUILDER = FLOAT_BLOCK.nestedClass("Builder"); static final ClassName ELEMENT_TYPE = ClassName.get(DATA_PACKAGE, "ElementType"); @@ -55,35 +57,41 @@ public class Types { static final ClassName INT_VECTOR = ClassName.get(DATA_PACKAGE, "IntVector"); static final ClassName LONG_VECTOR = ClassName.get(DATA_PACKAGE, "LongVector"); static final ClassName DOUBLE_VECTOR = ClassName.get(DATA_PACKAGE, "DoubleVector"); + static final ClassName FLOAT_VECTOR = ClassName.get(DATA_PACKAGE, "FloatVector"); static final ClassName BOOLEAN_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "BooleanVector", "Builder"); static final ClassName BYTES_REF_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "BytesRefVector", "Builder"); static final ClassName INT_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "IntVector", "Builder"); static final ClassName LONG_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "LongVector", "Builder"); static final ClassName DOUBLE_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "DoubleVector", "Builder"); + static final ClassName FLOAT_VECTOR_BUILDER = ClassName.get(DATA_PACKAGE, "FloatVector", "Builder"); static final ClassName BOOLEAN_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "BooleanVector", "FixedBuilder"); static final ClassName INT_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "IntVector", "FixedBuilder"); static final ClassName LONG_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "LongVector", "FixedBuilder"); static final ClassName DOUBLE_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "DoubleVector", "FixedBuilder"); + static final ClassName FLOAT_VECTOR_FIXED_BUILDER = ClassName.get(DATA_PACKAGE, "FloatVector", "FixedBuilder"); static final ClassName BOOLEAN_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "BooleanArrayVector"); static final ClassName BYTES_REF_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "BytesRefArrayVector"); static final ClassName INT_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "IntArrayVector"); static final ClassName LONG_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "LongArrayVector"); static final ClassName DOUBLE_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "DoubleArrayVector"); + static final ClassName FLOAT_ARRAY_VECTOR = ClassName.get(DATA_PACKAGE, "FloatArrayVector"); static final ClassName BOOLEAN_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "BooleanArrayBlock"); static final ClassName BYTES_REF_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "BytesRefArrayBlock"); static final ClassName INT_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "IntArrayBlock"); static final ClassName LONG_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "LongArrayBlock"); static final ClassName DOUBLE_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "DoubleArrayBlock"); + static final ClassName FLOAT_ARRAY_BLOCK = ClassName.get(DATA_PACKAGE, "FloatArrayBlock"); static final ClassName BOOLEAN_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantBooleanVector"); static final ClassName BYTES_REF_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantBytesRefVector"); static final ClassName INT_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantIntVector"); static final ClassName LONG_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantLongVector"); static final ClassName DOUBLE_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantDoubleVector"); + static final ClassName FLOAT_CONSTANT_VECTOR = ClassName.get(DATA_PACKAGE, "ConstantFloatVector"); static final ClassName AGGREGATOR_FUNCTION = ClassName.get(AGGREGATION_PACKAGE, "AggregatorFunction"); static final ClassName AGGREGATOR_FUNCTION_SUPPLIER = ClassName.get(AGGREGATION_PACKAGE, "AggregatorFunctionSupplier"); @@ -162,6 +170,9 @@ static ClassName blockType(String elementType) { if (elementType.equalsIgnoreCase(TypeName.DOUBLE.toString())) { return DOUBLE_BLOCK; } + if (elementType.equalsIgnoreCase(TypeName.FLOAT.toString())) { + return FLOAT_BLOCK; + } throw new IllegalArgumentException("unknown vector type for [" + elementType + "]"); } @@ -181,6 +192,9 @@ static ClassName vectorType(TypeName elementType) { if (elementType.equals(TypeName.DOUBLE)) { return DOUBLE_VECTOR; } + if (elementType.equals(TypeName.FLOAT)) { + return FLOAT_VECTOR; + } throw new IllegalArgumentException("unknown vector type for [" + elementType + "]"); } @@ -200,6 +214,9 @@ static ClassName vectorType(String elementType) { if (elementType.equalsIgnoreCase(TypeName.DOUBLE.toString())) { return DOUBLE_VECTOR; } + if (elementType.equalsIgnoreCase(TypeName.FLOAT.toString())) { + return FLOAT_VECTOR; + } throw new IllegalArgumentException("unknown vector type for [" + elementType + "]"); } @@ -234,6 +251,12 @@ static ClassName builderType(TypeName resultType) { if (resultType.equals(DOUBLE_VECTOR)) { return DOUBLE_VECTOR_BUILDER; } + if (resultType.equals(FLOAT_BLOCK)) { + return FLOAT_BLOCK_BUILDER; + } + if (resultType.equals(FLOAT_VECTOR)) { + return FLOAT_VECTOR_BUILDER; + } throw new IllegalArgumentException("unknown builder type for [" + resultType + "]"); } @@ -250,6 +273,9 @@ static ClassName vectorFixedBuilderType(TypeName elementType) { if (elementType.equals(TypeName.DOUBLE)) { return DOUBLE_VECTOR_FIXED_BUILDER; } + if (elementType.equals(TypeName.FLOAT)) { + return FLOAT_VECTOR_FIXED_BUILDER; + } throw new IllegalArgumentException("unknown vector fixed builder type for [" + elementType + "]"); } diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatArrayState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatArrayState.java new file mode 100644 index 0000000000000..b3767828f00db --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatArrayState.java @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.FloatArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasables; + +/** + * Aggregator state for an array of floats. It is created in a mode where it + * won't track the {@code groupId}s that are sent to it and it is the + * responsibility of the caller to only fetch values for {@code groupId}s + * that it has sent using the {@code selected} parameter when building the + * results. This is fine when there are no {@code null} values in the input + * data. But once there are null values in the input data it is + * much more convenient to only send non-null values and + * the tracking built into the grouping code can't track that. In that case + * call {@link #enableGroupIdTracking} to transition the state into a mode + * where it'll track which {@code groupIds} have been written. + *

    + * This class is generated. Do not edit it. + *

    + */ +final class FloatArrayState extends AbstractArrayState implements GroupingAggregatorState { + private final float init; + + private FloatArray values; + + FloatArrayState(BigArrays bigArrays, float init) { + super(bigArrays); + this.values = bigArrays.newFloatArray(1, false); + this.values.set(0, init); + this.init = init; + } + + float get(int groupId) { + return values.get(groupId); + } + + float getOrDefault(int groupId) { + return groupId < values.size() ? values.get(groupId) : init; + } + + void set(int groupId, float value) { + ensureCapacity(groupId); + values.set(groupId, value); + trackGroupId(groupId); + } + + Block toValuesBlock(org.elasticsearch.compute.data.IntVector selected, DriverContext driverContext) { + if (false == trackingGroupIds()) { + try (var builder = driverContext.blockFactory().newFloatVectorFixedBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + builder.appendFloat(i, values.get(selected.getInt(i))); + } + return builder.build().asBlock(); + } + } + try (FloatBlock.Builder builder = driverContext.blockFactory().newFloatBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (hasValue(group)) { + builder.appendFloat(values.get(group)); + } else { + builder.appendNull(); + } + } + return builder.build(); + } + } + + private void ensureCapacity(int groupId) { + if (groupId >= values.size()) { + long prevSize = values.size(); + values = bigArrays.grow(values, groupId + 1); + values.fill(prevSize, values.size(), init); + } + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate( + Block[] blocks, + int offset, + IntVector selected, + org.elasticsearch.compute.operator.DriverContext driverContext + ) { + assert blocks.length >= offset + 2; + try ( + var valuesBuilder = driverContext.blockFactory().newFloatBlockBuilder(selected.getPositionCount()); + var hasValueBuilder = driverContext.blockFactory().newBooleanVectorFixedBuilder(selected.getPositionCount()) + ) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + if (group < values.size()) { + valuesBuilder.appendFloat(values.get(group)); + } else { + valuesBuilder.appendFloat(0); // TODO can we just use null? + } + hasValueBuilder.appendBoolean(i, hasValue(group)); + } + blocks[offset + 0] = valuesBuilder.build(); + blocks[offset + 1] = hasValueBuilder.build().asBlock(); + } + } + + @Override + public void close() { + Releasables.close(values, super::close); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java new file mode 100644 index 0000000000000..81bdd39e51b6e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/FloatState.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * Aggregator state for a single float. + * This class is generated. Do not edit it. + */ +final class FloatState implements AggregatorState { + private float value; + private boolean seen; + + FloatState() { + this(0); + } + + FloatState(float init) { + this.value = init; + } + + float floatValue() { + return value; + } + + void floatValue(float value) { + this.value = value; + } + + boolean seen() { + return seen; + } + + void seen(boolean seen) { + this.seen = seen; + } + + /** Extracts an intermediate view of the contents of this state. */ + @Override + public void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + assert blocks.length >= offset + 2; + blocks[offset + 0] = driverContext.blockFactory().newConstantFloatBlockWith(value, 1); + blocks[offset + 1] = driverContext.blockFactory().newConstantBooleanBlockWith(seen, 1); + } + + @Override + public void close() {} +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java new file mode 100644 index 0000000000000..b50b125d98331 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/RateFloatAggregator.java @@ -0,0 +1,341 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ObjectArray; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; + +import java.util.Arrays; + +/** + * A rate grouping aggregation definition for float. + * This class is generated. Edit `X-RateAggregator.java.st` instead. + */ +@GroupingAggregator( + includeTimestamps = true, + value = { + @IntermediateState(name = "timestamps", type = "LONG_BLOCK"), + @IntermediateState(name = "values", type = "FLOAT_BLOCK"), + @IntermediateState(name = "resets", type = "DOUBLE") } +) +public class RateFloatAggregator { + + public static FloatRateGroupingState initGrouping(DriverContext driverContext, long unitInMillis) { + return new FloatRateGroupingState(driverContext.bigArrays(), driverContext.breaker(), unitInMillis); + } + + public static void combine(FloatRateGroupingState current, int groupId, long timestamp, float value) { + current.append(groupId, timestamp, value); + } + + public static void combineIntermediate( + FloatRateGroupingState current, + int groupId, + LongBlock timestamps, + FloatBlock values, + double reset, + int otherPosition + ) { + current.combine(groupId, timestamps, values, reset, otherPosition); + } + + public static void combineStates( + FloatRateGroupingState current, + int currentGroupId, // make the stylecheck happy + FloatRateGroupingState otherState, + int otherGroupId + ) { + current.combineState(currentGroupId, otherState, otherGroupId); + } + + public static Block evaluateFinal(FloatRateGroupingState state, IntVector selected, DriverContext driverContext) { + return state.evaluateFinal(selected, driverContext.blockFactory()); + } + + private static class FloatRateState { + static final long BASE_RAM_USAGE = RamUsageEstimator.sizeOfObject(FloatRateState.class); + final long[] timestamps; // descending order + final float[] values; + double reset = 0; + + FloatRateState(int initialSize) { + this.timestamps = new long[initialSize]; + this.values = new float[initialSize]; + } + + FloatRateState(long[] ts, float[] vs) { + this.timestamps = ts; + this.values = vs; + } + + private float dv(float v0, float v1) { + // counter reset detection + return v0 > v1 ? v1 : v1 - v0; + } + + void append(long t, float v) { + assert timestamps.length == 2 : "expected two timestamps; got " + timestamps.length; + assert t < timestamps[1] : "@timestamp goes backward: " + t + " >= " + timestamps[1]; + reset += dv(v, values[1]) + dv(values[1], values[0]) - dv(v, values[0]); + timestamps[1] = t; + values[1] = v; + } + + int entries() { + return timestamps.length; + } + + static long bytesUsed(int entries) { + var ts = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Long.BYTES * entries); + var vs = RamUsageEstimator.alignObjectSize(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) Float.BYTES * entries); + return BASE_RAM_USAGE + ts + vs; + } + } + + public static final class FloatRateGroupingState implements Releasable, Accountable, GroupingAggregatorState { + private ObjectArray states; + private final long unitInMillis; + private final BigArrays bigArrays; + private final CircuitBreaker breaker; + private long stateBytes; // for individual states + + FloatRateGroupingState(BigArrays bigArrays, CircuitBreaker breaker, long unitInMillis) { + this.bigArrays = bigArrays; + this.breaker = breaker; + this.states = bigArrays.newObjectArray(1); + this.unitInMillis = unitInMillis; + } + + void ensureCapacity(int groupId) { + states = bigArrays.grow(states, groupId + 1); + } + + void adjustBreaker(long bytes) { + breaker.addEstimateBytesAndMaybeBreak(bytes, "<>"); + stateBytes += bytes; + assert stateBytes >= 0 : stateBytes; + } + + void append(int groupId, long timestamp, float value) { + ensureCapacity(groupId); + var state = states.get(groupId); + if (state == null) { + adjustBreaker(FloatRateState.bytesUsed(1)); + state = new FloatRateState(new long[] { timestamp }, new float[] { value }); + states.set(groupId, state); + } else { + if (state.entries() == 1) { + adjustBreaker(FloatRateState.bytesUsed(2)); + state = new FloatRateState(new long[] { state.timestamps[0], timestamp }, new float[] { state.values[0], value }); + states.set(groupId, state); + adjustBreaker(-FloatRateState.bytesUsed(1)); // old state + } else { + state.append(timestamp, value); + } + } + } + + void combine(int groupId, LongBlock timestamps, FloatBlock values, double reset, int otherPosition) { + final int valueCount = timestamps.getValueCount(otherPosition); + if (valueCount == 0) { + return; + } + final int firstIndex = timestamps.getFirstValueIndex(otherPosition); + ensureCapacity(groupId); + var state = states.get(groupId); + if (state == null) { + adjustBreaker(FloatRateState.bytesUsed(valueCount)); + state = new FloatRateState(valueCount); + state.reset = reset; + states.set(groupId, state); + // TODO: add bulk_copy to Block + for (int i = 0; i < valueCount; i++) { + state.timestamps[i] = timestamps.getLong(firstIndex + i); + state.values[i] = values.getFloat(firstIndex + i); + } + } else { + adjustBreaker(FloatRateState.bytesUsed(state.entries() + valueCount)); + var newState = new FloatRateState(state.entries() + valueCount); + newState.reset = state.reset + reset; + states.set(groupId, newState); + merge(state, newState, firstIndex, valueCount, timestamps, values); + adjustBreaker(-FloatRateState.bytesUsed(state.entries())); // old state + } + } + + void merge(FloatRateState curr, FloatRateState dst, int firstIndex, int rightCount, LongBlock timestamps, FloatBlock values) { + int i = 0, j = 0, k = 0; + final int leftCount = curr.entries(); + while (i < leftCount && j < rightCount) { + final var t1 = curr.timestamps[i]; + final var t2 = timestamps.getLong(firstIndex + j); + if (t1 > t2) { + dst.timestamps[k] = t1; + dst.values[k] = curr.values[i]; + ++i; + } else { + dst.timestamps[k] = t2; + dst.values[k] = values.getFloat(firstIndex + j); + ++j; + } + ++k; + } + if (i < leftCount) { + System.arraycopy(curr.timestamps, i, dst.timestamps, k, leftCount - i); + System.arraycopy(curr.values, i, dst.values, k, leftCount - i); + } + while (j < rightCount) { + dst.timestamps[k] = timestamps.getLong(firstIndex + j); + dst.values[k] = values.getFloat(firstIndex + j); + ++k; + ++j; + } + } + + void combineState(int groupId, FloatRateGroupingState otherState, int otherGroupId) { + var other = otherGroupId < otherState.states.size() ? otherState.states.get(otherGroupId) : null; + if (other == null) { + return; + } + ensureCapacity(groupId); + var curr = states.get(groupId); + if (curr == null) { + var len = other.entries(); + adjustBreaker(FloatRateState.bytesUsed(len)); + curr = new FloatRateState(Arrays.copyOf(other.timestamps, len), Arrays.copyOf(other.values, len)); + curr.reset = other.reset; + states.set(groupId, curr); + } else { + states.set(groupId, mergeState(curr, other)); + } + } + + FloatRateState mergeState(FloatRateState s1, FloatRateState s2) { + var newLen = s1.entries() + s2.entries(); + adjustBreaker(FloatRateState.bytesUsed(newLen)); + var dst = new FloatRateState(newLen); + dst.reset = s1.reset + s2.reset; + int i = 0, j = 0, k = 0; + while (i < s1.entries() && j < s2.entries()) { + if (s1.timestamps[i] > s2.timestamps[j]) { + dst.timestamps[k] = s1.timestamps[i]; + dst.values[k] = s1.values[i]; + ++i; + } else { + dst.timestamps[k] = s2.timestamps[j]; + dst.values[k] = s2.values[j]; + ++j; + } + ++k; + } + System.arraycopy(s1.timestamps, i, dst.timestamps, k, s1.entries() - i); + System.arraycopy(s1.values, i, dst.values, k, s1.entries() - i); + System.arraycopy(s2.timestamps, j, dst.timestamps, k, s2.entries() - j); + System.arraycopy(s2.values, j, dst.values, k, s2.entries() - j); + return dst; + } + + @Override + public long ramBytesUsed() { + return states.ramBytesUsed() + stateBytes; + } + + @Override + public void close() { + Releasables.close(states, () -> adjustBreaker(-stateBytes)); + } + + @Override + public void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + assert blocks.length >= offset + 3 : "blocks=" + blocks.length + ",offset=" + offset; + final BlockFactory blockFactory = driverContext.blockFactory(); + final int positionCount = selected.getPositionCount(); + try ( + LongBlock.Builder timestamps = blockFactory.newLongBlockBuilder(positionCount * 2); + FloatBlock.Builder values = blockFactory.newFloatBlockBuilder(positionCount * 2); + DoubleVector.FixedBuilder resets = blockFactory.newDoubleVectorFixedBuilder(positionCount) + ) { + for (int i = 0; i < positionCount; i++) { + final var groupId = selected.getInt(i); + final var state = groupId < states.size() ? states.get(groupId) : null; + if (state != null) { + timestamps.beginPositionEntry(); + for (long t : state.timestamps) { + timestamps.appendLong(t); + } + timestamps.endPositionEntry(); + + values.beginPositionEntry(); + for (float v : state.values) { + values.appendFloat(v); + } + values.endPositionEntry(); + + resets.appendDouble(i, state.reset); + } else { + timestamps.appendNull(); + values.appendNull(); + resets.appendDouble(i, 0); + } + } + blocks[offset] = timestamps.build(); + blocks[offset + 1] = values.build(); + blocks[offset + 2] = resets.build().asBlock(); + } + } + + Block evaluateFinal(IntVector selected, BlockFactory blockFactory) { + int positionCount = selected.getPositionCount(); + try (DoubleBlock.Builder rates = blockFactory.newDoubleBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + final var groupId = selected.getInt(p); + final var state = groupId < states.size() ? states.get(groupId) : null; + if (state == null) { + rates.appendNull(); + continue; + } + int len = state.entries(); + long dt = state.timestamps[0] - state.timestamps[len - 1]; + if (dt == 0) { + // TODO: maybe issue warning when we don't have enough sample? + rates.appendNull(); + } else { + double reset = state.reset; + for (int i = 1; i < len; i++) { + if (state.values[i - 1] < state.values[i]) { + reset += state.values[i]; + } + } + double dv = state.values[0] - state.values[len - 1] + reset; + rates.appendDouble(dv * unitInMillis / dt); + } + } + return rates.build(); + } + } + + void enableGroupIdTracking(SeenGroupIds seenGroupIds) { + // noop - we handle the null states inside `toIntermediate` and `evaluateFinal` + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListDoubleAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListDoubleAggregator.java new file mode 100644 index 0000000000000..941722b4424d3 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListDoubleAggregator.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.sort.DoubleBucketedSort; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +/** + * Aggregates the top N field values for double. + */ +@Aggregator({ @IntermediateState(name = "topList", type = "DOUBLE_BLOCK") }) +@GroupingAggregator +class TopListDoubleAggregator { + public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { + return new SingleState(bigArrays, limit, ascending); + } + + public static void combine(SingleState state, double v) { + state.add(v); + } + + public static void combineIntermediate(SingleState state, DoubleBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.getDouble(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) { + return new GroupingState(bigArrays, limit, ascending); + } + + public static void combine(GroupingState state, int groupId, double v) { + state.add(groupId, v); + } + + public static void combineIntermediate(GroupingState state, int groupId, DoubleBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getDouble(i)); + } + } + + public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { + current.merge(groupId, state, statePosition); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class GroupingState implements Releasable { + private final DoubleBucketedSort sort; + + private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { + this.sort = new DoubleBucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + } + + public void add(int groupId, double value) { + sort.collect(value, groupId); + } + + public void merge(int groupId, GroupingState other, int otherGroupId) { + sort.merge(groupId, other.sort, otherGroupId); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + Releasables.closeExpectNoException(sort); + } + } + + public static class SingleState implements Releasable { + private final GroupingState internalState; + + private SingleState(BigArrays bigArrays, int limit, boolean ascending) { + this.internalState = new GroupingState(bigArrays, limit, ascending); + } + + public void add(double value) { + internalState.add(0, value); + } + + public void merge(GroupingState other) { + internalState.merge(0, other, 0); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + try (var intValues = blockFactory.newConstantIntVector(0, 1)) { + return internalState.toBlock(blockFactory, intValues); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListFloatAggregator.java new file mode 100644 index 0000000000000..c5fc51d5ba13f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListFloatAggregator.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.sort.FloatBucketedSort; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +/** + * Aggregates the top N field values for float. + */ +@Aggregator({ @IntermediateState(name = "topList", type = "FLOAT_BLOCK") }) +@GroupingAggregator +class TopListFloatAggregator { + public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { + return new SingleState(bigArrays, limit, ascending); + } + + public static void combine(SingleState state, float v) { + state.add(v); + } + + public static void combineIntermediate(SingleState state, FloatBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.getFloat(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) { + return new GroupingState(bigArrays, limit, ascending); + } + + public static void combine(GroupingState state, int groupId, float v) { + state.add(groupId, v); + } + + public static void combineIntermediate(GroupingState state, int groupId, FloatBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getFloat(i)); + } + } + + public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { + current.merge(groupId, state, statePosition); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class GroupingState implements Releasable { + private final FloatBucketedSort sort; + + private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { + this.sort = new FloatBucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + } + + public void add(int groupId, float value) { + sort.collect(value, groupId); + } + + public void merge(int groupId, GroupingState other, int otherGroupId) { + sort.merge(groupId, other.sort, otherGroupId); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + Releasables.closeExpectNoException(sort); + } + } + + public static class SingleState implements Releasable { + private final GroupingState internalState; + + private SingleState(BigArrays bigArrays, int limit, boolean ascending) { + this.internalState = new GroupingState(bigArrays, limit, ascending); + } + + public void add(float value) { + internalState.add(0, value); + } + + public void merge(GroupingState other) { + internalState.merge(0, other, 0); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + try (var intValues = blockFactory.newConstantIntVector(0, 1)) { + return internalState.toBlock(blockFactory, intValues); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListIntAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListIntAggregator.java new file mode 100644 index 0000000000000..dafbf1c2a3051 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListIntAggregator.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.sort.IntBucketedSort; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +/** + * Aggregates the top N field values for int. + */ +@Aggregator({ @IntermediateState(name = "topList", type = "INT_BLOCK") }) +@GroupingAggregator +class TopListIntAggregator { + public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { + return new SingleState(bigArrays, limit, ascending); + } + + public static void combine(SingleState state, int v) { + state.add(v); + } + + public static void combineIntermediate(SingleState state, IntBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.getInt(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) { + return new GroupingState(bigArrays, limit, ascending); + } + + public static void combine(GroupingState state, int groupId, int v) { + state.add(groupId, v); + } + + public static void combineIntermediate(GroupingState state, int groupId, IntBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getInt(i)); + } + } + + public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { + current.merge(groupId, state, statePosition); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class GroupingState implements Releasable { + private final IntBucketedSort sort; + + private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { + this.sort = new IntBucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + } + + public void add(int groupId, int value) { + sort.collect(value, groupId); + } + + public void merge(int groupId, GroupingState other, int otherGroupId) { + sort.merge(groupId, other.sort, otherGroupId); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + Releasables.closeExpectNoException(sort); + } + } + + public static class SingleState implements Releasable { + private final GroupingState internalState; + + private SingleState(BigArrays bigArrays, int limit, boolean ascending) { + this.internalState = new GroupingState(bigArrays, limit, ascending); + } + + public void add(int value) { + internalState.add(0, value); + } + + public void merge(GroupingState other) { + internalState.merge(0, other, 0); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + try (var intValues = blockFactory.newConstantIntVector(0, 1)) { + return internalState.toBlock(blockFactory, intValues); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListLongAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListLongAggregator.java new file mode 100644 index 0000000000000..c0e7122a4be0b --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/TopListLongAggregator.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.sort.LongBucketedSort; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +/** + * Aggregates the top N field values for long. + */ +@Aggregator({ @IntermediateState(name = "topList", type = "LONG_BLOCK") }) +@GroupingAggregator +class TopListLongAggregator { + public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { + return new SingleState(bigArrays, limit, ascending); + } + + public static void combine(SingleState state, long v) { + state.add(v); + } + + public static void combineIntermediate(SingleState state, LongBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.getLong(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) { + return new GroupingState(bigArrays, limit, ascending); + } + + public static void combine(GroupingState state, int groupId, long v) { + state.add(groupId, v); + } + + public static void combineIntermediate(GroupingState state, int groupId, LongBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getLong(i)); + } + } + + public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { + current.merge(groupId, state, statePosition); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class GroupingState implements Releasable { + private final LongBucketedSort sort; + + private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { + this.sort = new LongBucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + } + + public void add(int groupId, long value) { + sort.collect(value, groupId); + } + + public void merge(int groupId, GroupingState other, int otherGroupId) { + sort.merge(groupId, other.sort, otherGroupId); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + Releasables.closeExpectNoException(sort); + } + } + + public static class SingleState implements Releasable { + private final GroupingState internalState; + + private SingleState(BigArrays bigArrays, int limit, boolean ascending) { + this.internalState = new GroupingState(bigArrays, limit, ascending); + } + + public void add(long value) { + internalState.add(0, value); + } + + public void merge(GroupingState other) { + internalState.merge(0, other, 0); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + try (var intValues = blockFactory.newConstantIntVector(0, 1)) { + return internalState.toBlock(blockFactory, intValues); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java new file mode 100644 index 0000000000000..f9e5e1b7b283a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/aggregation/ValuesFloatAggregator.java @@ -0,0 +1,187 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.LongHash; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; + +/** + * Aggregates field values for float. + * This class is generated. Edit @{code X-ValuesAggregator.java.st} instead + * of this file. + */ +@Aggregator({ @IntermediateState(name = "values", type = "FLOAT_BLOCK") }) +@GroupingAggregator +class ValuesFloatAggregator { + public static SingleState initSingle(BigArrays bigArrays) { + return new SingleState(bigArrays); + } + + public static void combine(SingleState state, float v) { + state.values.add(Float.floatToIntBits(v)); + } + + public static void combineIntermediate(SingleState state, FloatBlock values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.getFloat(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays) { + return new GroupingState(bigArrays); + } + + public static void combine(GroupingState state, int groupId, float v) { + /* + * Encode the groupId and value into a single long - + * the top 32 bits for the group, the bottom 32 for the value. + */ + state.values.add((((long) groupId) << Float.SIZE) | (Float.floatToIntBits(v) & 0xFFFFFFFFL)); + } + + public static void combineIntermediate(GroupingState state, int groupId, FloatBlock values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.getFloat(i)); + } + } + + public static void combineStates(GroupingState current, int currentGroupId, GroupingState state, int statePosition) { + for (int id = 0; id < state.values.size(); id++) { + long both = state.values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group == statePosition) { + float value = Float.intBitsToFloat((int) both); + combine(current, currentGroupId, value); + } + } + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class SingleState implements Releasable { + private final LongHash values; + + private SingleState(BigArrays bigArrays) { + values = new LongHash(1, bigArrays); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + if (values.size() == 0) { + return blockFactory.newConstantNullBlock(1); + } + if (values.size() == 1) { + return blockFactory.newConstantFloatBlockWith(Float.intBitsToFloat((int) values.get(0)), 1); + } + try (FloatBlock.Builder builder = blockFactory.newFloatBlockBuilder((int) values.size())) { + builder.beginPositionEntry(); + for (int id = 0; id < values.size(); id++) { + builder.appendFloat(Float.intBitsToFloat((int) values.get(id))); + } + builder.endPositionEntry(); + return builder.build(); + } + } + + @Override + public void close() { + values.close(); + } + } + + /** + * State for a grouped {@code VALUES} aggregation. This implementation + * emphasizes collect-time performance over the performance of rendering + * results. That's good, but it's a pretty intensive emphasis, requiring + * an {@code O(n^2)} operation for collection to support a {@code O(1)} + * collector operation. But at least it's fairly simple. + */ + public static class GroupingState implements Releasable { + private final LongHash values; + + private GroupingState(BigArrays bigArrays) { + values = new LongHash(1, bigArrays); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + if (values.size() == 0) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + try (FloatBlock.Builder builder = blockFactory.newFloatBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int selectedGroup = selected.getInt(s); + /* + * Count can effectively be in three states - 0, 1, many. We use those + * states to buffer the first value, so we can avoid calling + * beginPositionEntry on single valued fields. + */ + int count = 0; + float first = 0; + for (int id = 0; id < values.size(); id++) { + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group == selectedGroup) { + float value = Float.intBitsToFloat((int) both); + switch (count) { + case 0 -> first = value; + case 1 -> { + builder.beginPositionEntry(); + builder.appendFloat(first); + builder.appendFloat(value); + } + default -> builder.appendFloat(value); + } + count++; + } + } + switch (count) { + case 0 -> builder.appendNull(); + case 1 -> builder.appendFloat(first); + default -> builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + values.close(); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantFloatVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantFloatVector.java new file mode 100644 index 0000000000000..2e674288eac92 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/ConstantFloatVector.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.ReleasableIterator; + +/** + * Vector implementation that stores a constant float value. + * This class is generated. Do not edit it. + */ +final class ConstantFloatVector extends AbstractVector implements FloatVector { + + static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ConstantFloatVector.class); + + private final float value; + + ConstantFloatVector(float value, int positionCount, BlockFactory blockFactory) { + super(positionCount, blockFactory); + this.value = value; + } + + @Override + public float getFloat(int position) { + return value; + } + + @Override + public FloatBlock asBlock() { + return new FloatVectorBlock(this); + } + + @Override + public FloatVector filter(int... positions) { + return blockFactory().newConstantFloatVector(value, positions.length); + } + + @Override + public ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize) { + if (positions.getPositionCount() == 0) { + return ReleasableIterator.empty(); + } + IntVector positionsVector = positions.asVector(); + if (positionsVector == null) { + return new FloatLookup(asBlock(), positions, targetBlockSize); + } + int min = positionsVector.min(); + if (min < 0) { + throw new IllegalArgumentException("invalid position [" + min + "]"); + } + if (min > getPositionCount()) { + return ReleasableIterator.single((FloatBlock) positions.blockFactory().newConstantNullBlock(positions.getPositionCount())); + } + if (positionsVector.max() < getPositionCount()) { + return ReleasableIterator.single(positions.blockFactory().newConstantFloatBlockWith(value, positions.getPositionCount())); + } + return new FloatLookup(asBlock(), positions, targetBlockSize); + } + + @Override + public ElementType elementType() { + return ElementType.FLOAT; + } + + @Override + public boolean isConstant() { + return true; + } + + @Override + public long ramBytesUsed() { + return RAM_BYTES_USED; + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FloatVector that) { + return FloatVector.equals(this, that); + } + return false; + } + + @Override + public int hashCode() { + return FloatVector.hash(this); + } + + public String toString() { + return getClass().getSimpleName() + "[positions=" + getPositionCount() + ", value=" + value + ']'; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java new file mode 100644 index 0000000000000..b2666a9d86926 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayBlock.java @@ -0,0 +1,205 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.ReleasableIterator; +import org.elasticsearch.core.Releasables; + +import java.io.IOException; +import java.util.BitSet; + +/** + * Block implementation that stores values in a {@link FloatArrayVector}. + * This class is generated. Do not edit it. + */ +final class FloatArrayBlock extends AbstractArrayBlock implements FloatBlock { + + static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FloatArrayBlock.class); + + private final FloatArrayVector vector; + + FloatArrayBlock( + float[] values, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory + ) { + this( + new FloatArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering + ); + } + + private FloatArrayBlock( + FloatArrayVector vector, // stylecheck + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering + ) { + super(positionCount, firstValueIndexes, nulls, mvOrdering); + this.vector = vector; + assert firstValueIndexes == null + ? vector.getPositionCount() == getPositionCount() + : firstValueIndexes[getPositionCount()] == vector.getPositionCount(); + } + + static FloatArrayBlock readArrayBlock(BlockFactory blockFactory, BlockStreamInput in) throws IOException { + final SubFields sub = new SubFields(blockFactory, in); + FloatArrayVector vector = null; + boolean success = false; + try { + vector = FloatArrayVector.readArrayVector(sub.vectorPositions(), in, blockFactory); + var block = new FloatArrayBlock(vector, sub.positionCount, sub.firstValueIndexes, sub.nullsMask, sub.mvOrdering); + blockFactory.adjustBreaker(block.ramBytesUsed() - vector.ramBytesUsed() - sub.bytesReserved); + success = true; + return block; + } finally { + if (success == false) { + Releasables.close(vector); + blockFactory.adjustBreaker(-sub.bytesReserved); + } + } + } + + void writeArrayBlock(StreamOutput out) throws IOException { + writeSubFields(out); + vector.writeArrayVector(vector.getPositionCount(), out); + } + + @Override + public FloatVector asVector() { + return null; + } + + @Override + public float getFloat(int valueIndex) { + return vector.getFloat(valueIndex); + } + + @Override + public FloatBlock filter(int... positions) { + try (var builder = blockFactory().newFloatBlockBuilder(positions.length)) { + for (int pos : positions) { + if (isNull(pos)) { + builder.appendNull(); + continue; + } + int valueCount = getValueCount(pos); + int first = getFirstValueIndex(pos); + if (valueCount == 1) { + builder.appendFloat(getFloat(getFirstValueIndex(pos))); + } else { + builder.beginPositionEntry(); + for (int c = 0; c < valueCount; c++) { + builder.appendFloat(getFloat(first + c)); + } + builder.endPositionEntry(); + } + } + return builder.mvOrdering(mvOrdering()).build(); + } + } + + @Override + public ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize) { + return new FloatLookup(this, positions, targetBlockSize); + } + + @Override + public ElementType elementType() { + return ElementType.FLOAT; + } + + @Override + public FloatBlock expand() { + if (firstValueIndexes == null) { + incRef(); + return this; + } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = Math.max(nullsMask.size(), BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount)); + blockFactory().adjustBreaker(bitSetRamUsedEstimate); + + FloatArrayBlock expanded = new FloatArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; + } + + private long ramBytesUsedOnlyBlock() { + return BASE_RAM_BYTES_USED + BlockRamUsageEstimator.sizeOf(firstValueIndexes) + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); + } + + @Override + public long ramBytesUsed() { + return ramBytesUsedOnlyBlock() + vector.ramBytesUsed(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FloatBlock that) { + return FloatBlock.equals(this, that); + } + return false; + } + + @Override + public int hashCode() { + return FloatBlock.hash(this); + } + + @Override + public String toString() { + return getClass().getSimpleName() + + "[positions=" + + getPositionCount() + + ", mvOrdering=" + + mvOrdering() + + ", vector=" + + vector + + ']'; + } + + @Override + public void allowPassingToDifferentDriver() { + vector.allowPassingToDifferentDriver(); + } + + @Override + public BlockFactory blockFactory() { + return vector.blockFactory(); + } + + @Override + public void closeInternal() { + blockFactory().adjustBreaker(-ramBytesUsedOnlyBlock()); + Releasables.closeExpectNoException(vector); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayVector.java new file mode 100644 index 0000000000000..7e5e0eef436ff --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatArrayVector.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.ReleasableIterator; + +import java.io.IOException; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +/** + * Vector implementation that stores an array of float values. + * This class is generated. Do not edit it. + */ +final class FloatArrayVector extends AbstractVector implements FloatVector { + + static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FloatArrayVector.class) + // TODO: remove these extra bytes once `asBlock` returns a block with a separate reference to the vector. + + RamUsageEstimator.shallowSizeOfInstance(FloatVectorBlock.class) + // TODO: remove this if/when we account for memory used by Pages + + Block.PAGE_MEM_OVERHEAD_PER_BLOCK; + + private final float[] values; + + FloatArrayVector(float[] values, int positionCount, BlockFactory blockFactory) { + super(positionCount, blockFactory); + this.values = values; + } + + static FloatArrayVector readArrayVector(int positions, StreamInput in, BlockFactory blockFactory) throws IOException { + final long preAdjustedBytes = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + (long) positions * Float.BYTES; + blockFactory.adjustBreaker(preAdjustedBytes); + boolean success = false; + try { + float[] values = new float[positions]; + for (int i = 0; i < positions; i++) { + values[i] = in.readFloat(); + } + final var block = new FloatArrayVector(values, positions, blockFactory); + blockFactory.adjustBreaker(block.ramBytesUsed() - preAdjustedBytes); + success = true; + return block; + } finally { + if (success == false) { + blockFactory.adjustBreaker(-preAdjustedBytes); + } + } + } + + void writeArrayVector(int positions, StreamOutput out) throws IOException { + for (int i = 0; i < positions; i++) { + out.writeFloat(values[i]); + } + } + + @Override + public FloatBlock asBlock() { + return new FloatVectorBlock(this); + } + + @Override + public float getFloat(int position) { + return values[position]; + } + + @Override + public ElementType elementType() { + return ElementType.FLOAT; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public FloatVector filter(int... positions) { + try (FloatVector.Builder builder = blockFactory().newFloatVectorBuilder(positions.length)) { + for (int pos : positions) { + builder.appendFloat(values[pos]); + } + return builder.build(); + } + } + + @Override + public ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize) { + return new FloatLookup(asBlock(), positions, targetBlockSize); + } + + public static long ramBytesEstimated(float[] values) { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + + @Override + public long ramBytesUsed() { + return ramBytesEstimated(values); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FloatVector that) { + return FloatVector.equals(this, that); + } + return false; + } + + @Override + public int hashCode() { + return FloatVector.hash(this); + } + + @Override + public String toString() { + String valuesString = IntStream.range(0, getPositionCount()) + .limit(10) + .mapToObj(n -> String.valueOf(values[n])) + .collect(Collectors.joining(", ", "[", getPositionCount() > 10 ? ", ...]" : "]")); + return getClass().getSimpleName() + "[positions=" + getPositionCount() + ", values=" + valuesString + ']'; + } + +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayBlock.java new file mode 100644 index 0000000000000..693823636043a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayBlock.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.FloatArray; +import org.elasticsearch.core.ReleasableIterator; +import org.elasticsearch.core.Releasables; + +import java.io.IOException; +import java.util.BitSet; + +/** + * Block implementation that stores values in a {@link FloatBigArrayVector}. Does not take ownership of the given + * {@link FloatArray} and does not adjust circuit breakers to account for it. + * This class is generated. Do not edit it. + */ +public final class FloatBigArrayBlock extends AbstractArrayBlock implements FloatBlock { + + private static final long BASE_RAM_BYTES_USED = 0; // TODO: fix this + private final FloatBigArrayVector vector; + + public FloatBigArrayBlock( + FloatArray values, + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering, + BlockFactory blockFactory + ) { + this( + new FloatBigArrayVector(values, firstValueIndexes == null ? positionCount : firstValueIndexes[positionCount], blockFactory), + positionCount, + firstValueIndexes, + nulls, + mvOrdering + ); + } + + private FloatBigArrayBlock( + FloatBigArrayVector vector, // stylecheck + int positionCount, + int[] firstValueIndexes, + BitSet nulls, + MvOrdering mvOrdering + ) { + super(positionCount, firstValueIndexes, nulls, mvOrdering); + this.vector = vector; + assert firstValueIndexes == null + ? vector.getPositionCount() == getPositionCount() + : firstValueIndexes[getPositionCount()] == vector.getPositionCount(); + } + + static FloatBigArrayBlock readArrayBlock(BlockFactory blockFactory, BlockStreamInput in) throws IOException { + final SubFields sub = new SubFields(blockFactory, in); + FloatBigArrayVector vector = null; + boolean success = false; + try { + vector = FloatBigArrayVector.readArrayVector(sub.vectorPositions(), in, blockFactory); + var block = new FloatBigArrayBlock(vector, sub.positionCount, sub.firstValueIndexes, sub.nullsMask, sub.mvOrdering); + blockFactory.adjustBreaker(block.ramBytesUsed() - vector.ramBytesUsed() - sub.bytesReserved); + success = true; + return block; + } finally { + if (success == false) { + Releasables.close(vector); + blockFactory.adjustBreaker(-sub.bytesReserved); + } + } + } + + void writeArrayBlock(StreamOutput out) throws IOException { + writeSubFields(out); + vector.writeArrayVector(vector.getPositionCount(), out); + } + + @Override + public FloatVector asVector() { + return null; + } + + @Override + public float getFloat(int valueIndex) { + return vector.getFloat(valueIndex); + } + + @Override + public FloatBlock filter(int... positions) { + try (var builder = blockFactory().newFloatBlockBuilder(positions.length)) { + for (int pos : positions) { + if (isNull(pos)) { + builder.appendNull(); + continue; + } + int valueCount = getValueCount(pos); + int first = getFirstValueIndex(pos); + if (valueCount == 1) { + builder.appendFloat(getFloat(getFirstValueIndex(pos))); + } else { + builder.beginPositionEntry(); + for (int c = 0; c < valueCount; c++) { + builder.appendFloat(getFloat(first + c)); + } + builder.endPositionEntry(); + } + } + return builder.mvOrdering(mvOrdering()).build(); + } + } + + @Override + public ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize) { + return new FloatLookup(this, positions, targetBlockSize); + } + + @Override + public ElementType elementType() { + return ElementType.FLOAT; + } + + @Override + public FloatBlock expand() { + if (firstValueIndexes == null) { + incRef(); + return this; + } + if (nullsMask == null) { + vector.incRef(); + return vector.asBlock(); + } + + // The following line is correct because positions with multi-values are never null. + int expandedPositionCount = vector.getPositionCount(); + long bitSetRamUsedEstimate = Math.max(nullsMask.size(), BlockRamUsageEstimator.sizeOfBitSet(expandedPositionCount)); + blockFactory().adjustBreaker(bitSetRamUsedEstimate); + + FloatBigArrayBlock expanded = new FloatBigArrayBlock( + vector, + expandedPositionCount, + null, + shiftNullsToExpandedPositions(), + MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING + ); + blockFactory().adjustBreaker(expanded.ramBytesUsedOnlyBlock() - bitSetRamUsedEstimate); + // We need to incRef after adjusting any breakers, otherwise we might leak the vector if the breaker trips. + vector.incRef(); + return expanded; + } + + private long ramBytesUsedOnlyBlock() { + return BASE_RAM_BYTES_USED + BlockRamUsageEstimator.sizeOf(firstValueIndexes) + BlockRamUsageEstimator.sizeOfBitSet(nullsMask); + } + + @Override + public long ramBytesUsed() { + return ramBytesUsedOnlyBlock() + RamUsageEstimator.sizeOf(vector); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FloatBlock that) { + return FloatBlock.equals(this, that); + } + return false; + } + + @Override + public int hashCode() { + return FloatBlock.hash(this); + } + + @Override + public String toString() { + return getClass().getSimpleName() + + "[positions=" + + getPositionCount() + + ", mvOrdering=" + + mvOrdering() + + ", ramBytesUsed=" + + vector.ramBytesUsed() + + ']'; + } + + @Override + public void allowPassingToDifferentDriver() { + vector.allowPassingToDifferentDriver(); + } + + @Override + public BlockFactory blockFactory() { + return vector.blockFactory(); + } + + @Override + public void closeInternal() { + blockFactory().adjustBreaker(-ramBytesUsedOnlyBlock()); + Releasables.closeExpectNoException(vector); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayVector.java new file mode 100644 index 0000000000000..2de1019103522 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBigArrayVector.java @@ -0,0 +1,108 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.common.util.FloatArray; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.ReleasableIterator; + +import java.io.IOException; + +/** + * Vector implementation that defers to an enclosed {@link FloatArray}. + * Does not take ownership of the array and does not adjust circuit breakers to account for it. + * This class is generated. Do not edit it. + */ +public final class FloatBigArrayVector extends AbstractVector implements FloatVector, Releasable { + + private static final long BASE_RAM_BYTES_USED = 0; // FIXME + + private final FloatArray values; + + public FloatBigArrayVector(FloatArray values, int positionCount, BlockFactory blockFactory) { + super(positionCount, blockFactory); + this.values = values; + } + + static FloatBigArrayVector readArrayVector(int positions, StreamInput in, BlockFactory blockFactory) throws IOException { + throw new UnsupportedOperationException(); + } + + void writeArrayVector(int positions, StreamOutput out) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public FloatBlock asBlock() { + return new FloatVectorBlock(this); + } + + @Override + public float getFloat(int position) { + return values.get(position); + } + + @Override + public ElementType elementType() { + return ElementType.FLOAT; + } + + @Override + public boolean isConstant() { + return false; + } + + @Override + public long ramBytesUsed() { + return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values); + } + + @Override + public FloatVector filter(int... positions) { + var blockFactory = blockFactory(); + final FloatArray filtered = blockFactory.bigArrays().newFloatArray(positions.length); + for (int i = 0; i < positions.length; i++) { + filtered.set(i, values.get(positions[i])); + } + return new FloatBigArrayVector(filtered, positions.length, blockFactory); + } + + @Override + public ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize) { + return new FloatLookup(asBlock(), positions, targetBlockSize); + } + + @Override + public void closeInternal() { + // The circuit breaker that tracks the values {@link FloatArray} is adjusted outside + // of this class. + values.close(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FloatVector that) { + return FloatVector.equals(this, that); + } + return false; + } + + @Override + public int hashCode() { + return FloatVector.hash(this); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[positions=" + getPositionCount() + ", values=" + values + ']'; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java new file mode 100644 index 0000000000000..3d2def604a61e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlock.java @@ -0,0 +1,234 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.ReleasableIterator; +import org.elasticsearch.index.mapper.BlockLoader; + +import java.io.IOException; + +/** + * Block that stores float values. + * This class is generated. Do not edit it. + */ +public sealed interface FloatBlock extends Block permits FloatArrayBlock, FloatVectorBlock, ConstantNullBlock, FloatBigArrayBlock { + + /** + * Retrieves the float value stored at the given value index. + * + *

    Values for a given position are between getFirstValueIndex(position) (inclusive) and + * getFirstValueIndex(position) + getValueCount(position) (exclusive). + * + * @param valueIndex the value index + * @return the data value (as a float) + */ + float getFloat(int valueIndex); + + @Override + FloatVector asVector(); + + @Override + FloatBlock filter(int... positions); + + @Override + ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize); + + @Override + FloatBlock expand(); + + @Override + default String getWriteableName() { + return "FloatBlock"; + } + + NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Block.class, "FloatBlock", FloatBlock::readFrom); + + private static FloatBlock readFrom(StreamInput in) throws IOException { + return readFrom((BlockStreamInput) in); + } + + static FloatBlock readFrom(BlockStreamInput in) throws IOException { + final byte serializationType = in.readByte(); + return switch (serializationType) { + case SERIALIZE_BLOCK_VALUES -> FloatBlock.readValues(in); + case SERIALIZE_BLOCK_VECTOR -> FloatVector.readFrom(in.blockFactory(), in).asBlock(); + case SERIALIZE_BLOCK_ARRAY -> FloatArrayBlock.readArrayBlock(in.blockFactory(), in); + case SERIALIZE_BLOCK_BIG_ARRAY -> FloatBigArrayBlock.readArrayBlock(in.blockFactory(), in); + default -> { + assert false : "invalid block serialization type " + serializationType; + throw new IllegalStateException("invalid serialization type " + serializationType); + } + }; + } + + private static FloatBlock readValues(BlockStreamInput in) throws IOException { + final int positions = in.readVInt(); + try (FloatBlock.Builder builder = in.blockFactory().newFloatBlockBuilder(positions)) { + for (int i = 0; i < positions; i++) { + if (in.readBoolean()) { + builder.appendNull(); + } else { + final int valueCount = in.readVInt(); + builder.beginPositionEntry(); + for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) { + builder.appendFloat(in.readFloat()); + } + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + @Override + default void writeTo(StreamOutput out) throws IOException { + FloatVector vector = asVector(); + final var version = out.getTransportVersion(); + if (vector != null) { + out.writeByte(SERIALIZE_BLOCK_VECTOR); + vector.writeTo(out); + } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_BLOCK) && this instanceof FloatArrayBlock b) { + out.writeByte(SERIALIZE_BLOCK_ARRAY); + b.writeArrayBlock(out); + } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_ARRAY) && this instanceof FloatBigArrayBlock b) { + out.writeByte(SERIALIZE_BLOCK_BIG_ARRAY); + b.writeArrayBlock(out); + } else { + out.writeByte(SERIALIZE_BLOCK_VALUES); + FloatBlock.writeValues(this, out); + } + } + + private static void writeValues(FloatBlock block, StreamOutput out) throws IOException { + final int positions = block.getPositionCount(); + out.writeVInt(positions); + for (int pos = 0; pos < positions; pos++) { + if (block.isNull(pos)) { + out.writeBoolean(true); + } else { + out.writeBoolean(false); + final int valueCount = block.getValueCount(pos); + out.writeVInt(valueCount); + for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) { + out.writeFloat(block.getFloat(block.getFirstValueIndex(pos) + valueIndex)); + } + } + } + } + + /** + * Compares the given object with this block for equality. Returns {@code true} if and only if the + * given object is a FloatBlock, and both blocks are {@link #equals(FloatBlock, FloatBlock) equal}. + */ + @Override + boolean equals(Object obj); + + /** Returns the hash code of this block, as defined by {@link #hash(FloatBlock)}. */ + @Override + int hashCode(); + + /** + * Returns {@code true} if the given blocks are equal to each other, otherwise {@code false}. + * Two blocks are considered equal if they have the same position count, and contain the same + * values (including absent null values) in the same order. This definition ensures that the + * equals method works properly across different implementations of the FloatBlock interface. + */ + static boolean equals(FloatBlock block1, FloatBlock block2) { + if (block1 == block2) { + return true; + } + final int positions = block1.getPositionCount(); + if (positions != block2.getPositionCount()) { + return false; + } + for (int pos = 0; pos < positions; pos++) { + if (block1.isNull(pos) || block2.isNull(pos)) { + if (block1.isNull(pos) != block2.isNull(pos)) { + return false; + } + } else { + final int valueCount = block1.getValueCount(pos); + if (valueCount != block2.getValueCount(pos)) { + return false; + } + final int b1ValueIdx = block1.getFirstValueIndex(pos); + final int b2ValueIdx = block2.getFirstValueIndex(pos); + for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) { + if (block1.getFloat(b1ValueIdx + valueIndex) != block2.getFloat(b2ValueIdx + valueIndex)) { + return false; + } + } + } + } + return true; + } + + /** + * Generates the hash code for the given block. The hash code is computed from the block's values. + * This ensures that {@code block1.equals(block2)} implies that {@code block1.hashCode()==block2.hashCode()} + * for any two blocks, {@code block1} and {@code block2}, as required by the general contract of + * {@link Object#hashCode}. + */ + static int hash(FloatBlock block) { + final int positions = block.getPositionCount(); + int result = 1; + for (int pos = 0; pos < positions; pos++) { + if (block.isNull(pos)) { + result = 31 * result - 1; + } else { + final int valueCount = block.getValueCount(pos); + result = 31 * result + valueCount; + final int firstValueIdx = block.getFirstValueIndex(pos); + for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) { + result = 31 * result + Float.floatToIntBits(block.getFloat(pos)); + } + } + } + return result; + } + + /** + * Builder for {@link FloatBlock} + */ + sealed interface Builder extends Block.Builder, BlockLoader.FloatBuilder permits FloatBlockBuilder { + /** + * Appends a float to the current entry. + */ + @Override + Builder appendFloat(float value); + + /** + * Copy the values in {@code block} from {@code beginInclusive} to + * {@code endExclusive} into this builder. + */ + Builder copyFrom(FloatBlock block, int beginInclusive, int endExclusive); + + @Override + Builder appendNull(); + + @Override + Builder beginPositionEntry(); + + @Override + Builder endPositionEntry(); + + @Override + Builder copyFrom(Block block, int beginInclusive, int endExclusive); + + @Override + Builder mvOrdering(Block.MvOrdering mvOrdering); + + @Override + FloatBlock build(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlockBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlockBuilder.java new file mode 100644 index 0000000000000..9c1e7aba49a21 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatBlockBuilder.java @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.common.breaker.CircuitBreakingException; +import org.elasticsearch.common.util.FloatArray; + +import java.util.Arrays; + +/** + * Block build of FloatBlocks. + * This class is generated. Do not edit it. + */ +final class FloatBlockBuilder extends AbstractBlockBuilder implements FloatBlock.Builder { + + private float[] values; + + FloatBlockBuilder(int estimatedSize, BlockFactory blockFactory) { + super(blockFactory); + int initialSize = Math.max(estimatedSize, 2); + adjustBreaker(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + initialSize * elementSize()); + values = new float[initialSize]; + } + + @Override + public FloatBlockBuilder appendFloat(float value) { + ensureCapacity(); + values[valueCount] = value; + hasNonNullValue = true; + valueCount++; + updatePosition(); + return this; + } + + @Override + protected int elementSize() { + return Float.BYTES; + } + + @Override + protected int valuesLength() { + return values.length; + } + + @Override + protected void growValuesArray(int newSize) { + values = Arrays.copyOf(values, newSize); + } + + @Override + public FloatBlockBuilder appendNull() { + super.appendNull(); + return this; + } + + @Override + public FloatBlockBuilder beginPositionEntry() { + super.beginPositionEntry(); + return this; + } + + @Override + public FloatBlockBuilder endPositionEntry() { + super.endPositionEntry(); + return this; + } + + @Override + public FloatBlockBuilder copyFrom(Block block, int beginInclusive, int endExclusive) { + if (block.areAllValuesNull()) { + for (int p = beginInclusive; p < endExclusive; p++) { + appendNull(); + } + return this; + } + return copyFrom((FloatBlock) block, beginInclusive, endExclusive); + } + + /** + * Copy the values in {@code block} from {@code beginInclusive} to + * {@code endExclusive} into this builder. + */ + public FloatBlockBuilder copyFrom(FloatBlock block, int beginInclusive, int endExclusive) { + if (endExclusive > block.getPositionCount()) { + throw new IllegalArgumentException("can't copy past the end [" + endExclusive + " > " + block.getPositionCount() + "]"); + } + FloatVector vector = block.asVector(); + if (vector != null) { + copyFromVector(vector, beginInclusive, endExclusive); + } else { + copyFromBlock(block, beginInclusive, endExclusive); + } + return this; + } + + private void copyFromBlock(FloatBlock block, int beginInclusive, int endExclusive) { + for (int p = beginInclusive; p < endExclusive; p++) { + if (block.isNull(p)) { + appendNull(); + continue; + } + int count = block.getValueCount(p); + if (count > 1) { + beginPositionEntry(); + } + int i = block.getFirstValueIndex(p); + for (int v = 0; v < count; v++) { + appendFloat(block.getFloat(i++)); + } + if (count > 1) { + endPositionEntry(); + } + } + } + + private void copyFromVector(FloatVector vector, int beginInclusive, int endExclusive) { + for (int p = beginInclusive; p < endExclusive; p++) { + appendFloat(vector.getFloat(p)); + } + } + + @Override + public FloatBlockBuilder mvOrdering(Block.MvOrdering mvOrdering) { + this.mvOrdering = mvOrdering; + return this; + } + + private FloatBlock buildBigArraysBlock() { + final FloatBlock theBlock; + final FloatArray array = blockFactory.bigArrays().newFloatArray(valueCount, false); + for (int i = 0; i < valueCount; i++) { + array.set(i, values[i]); + } + if (isDense() && singleValued()) { + theBlock = new FloatBigArrayVector(array, positionCount, blockFactory).asBlock(); + } else { + theBlock = new FloatBigArrayBlock(array, positionCount, firstValueIndexes, nullsMask, mvOrdering, blockFactory); + } + /* + * Update the breaker with the actual bytes used. + * We pass false below even though we've used the bytes. That's weird, + * but if we break here we will throw away the used memory, letting + * it be deallocated. The exception will bubble up and the builder will + * still technically be open, meaning the calling code should close it + * which will return all used memory to the breaker. + */ + blockFactory.adjustBreaker(theBlock.ramBytesUsed() - estimatedBytes - array.ramBytesUsed()); + return theBlock; + } + + @Override + public FloatBlock build() { + try { + finish(); + FloatBlock theBlock; + if (hasNonNullValue && positionCount == 1 && valueCount == 1) { + theBlock = blockFactory.newConstantFloatBlockWith(values[0], 1, estimatedBytes); + } else if (estimatedBytes > blockFactory.maxPrimitiveArrayBytes()) { + theBlock = buildBigArraysBlock(); + } else if (isDense() && singleValued()) { + theBlock = blockFactory.newFloatArrayVector(values, positionCount, estimatedBytes).asBlock(); + } else { + theBlock = blockFactory.newFloatArrayBlock( + values, // stylecheck + positionCount, + firstValueIndexes, + nullsMask, + mvOrdering, + estimatedBytes + ); + } + built(); + return theBlock; + } catch (CircuitBreakingException e) { + close(); + throw e; + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatLookup.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatLookup.java new file mode 100644 index 0000000000000..9e0018e527c4d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatLookup.java @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.compute.operator.Operator; +import org.elasticsearch.core.ReleasableIterator; +import org.elasticsearch.core.Releasables; + +/** + * Generic {@link Block#lookup} implementation {@link FloatBlock}s. + * This class is generated. Do not edit it. + */ +final class FloatLookup implements ReleasableIterator { + private final FloatBlock values; + private final IntBlock positions; + private final long targetByteSize; + private int position; + + private float first; + private int valuesInPosition; + + FloatLookup(FloatBlock values, IntBlock positions, ByteSizeValue targetBlockSize) { + values.incRef(); + positions.incRef(); + this.values = values; + this.positions = positions; + this.targetByteSize = targetBlockSize.getBytes(); + } + + @Override + public boolean hasNext() { + return position < positions.getPositionCount(); + } + + @Override + public FloatBlock next() { + try (FloatBlock.Builder builder = positions.blockFactory().newFloatBlockBuilder(positions.getTotalValueCount())) { + int count = 0; + while (position < positions.getPositionCount()) { + int start = positions.getFirstValueIndex(position); + int end = start + positions.getValueCount(position); + valuesInPosition = 0; + for (int i = start; i < end; i++) { + copy(builder, positions.getInt(i)); + } + switch (valuesInPosition) { + case 0 -> builder.appendNull(); + case 1 -> builder.appendFloat(first); + default -> builder.endPositionEntry(); + } + position++; + // TOOD what if the estimate is super huge? should we break even with less than MIN_TARGET? + if (++count > Operator.MIN_TARGET_PAGE_SIZE && builder.estimatedBytes() < targetByteSize) { + break; + } + } + return builder.build(); + } + } + + private void copy(FloatBlock.Builder builder, int valuePosition) { + if (valuePosition >= values.getPositionCount()) { + return; + } + int start = values.getFirstValueIndex(valuePosition); + int end = start + values.getValueCount(valuePosition); + for (int i = start; i < end; i++) { + if (valuesInPosition == 0) { + first = values.getFloat(i); + valuesInPosition++; + continue; + } + if (valuesInPosition == 1) { + builder.beginPositionEntry(); + builder.appendFloat(first); + } + if (valuesInPosition > Block.MAX_LOOKUP) { + // TODO replace this with a warning and break + throw new IllegalArgumentException("Found a single entry with " + valuesInPosition + " entries"); + } + builder.appendFloat(values.getFloat(i)); + valuesInPosition++; + } + } + + @Override + public void close() { + Releasables.close(values, positions); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java new file mode 100644 index 0000000000000..5fd2ae7b9c719 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVector.java @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.elasticsearch.TransportVersions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.ReleasableIterator; + +import java.io.IOException; + +/** + * Vector that stores float values. + * This class is generated. Do not edit it. + */ +public sealed interface FloatVector extends Vector permits ConstantFloatVector, FloatArrayVector, FloatBigArrayVector, ConstantNullVector { + + float getFloat(int position); + + @Override + FloatBlock asBlock(); + + @Override + FloatVector filter(int... positions); + + @Override + ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize); + + /** + * Compares the given object with this vector for equality. Returns {@code true} if and only if the + * given object is a FloatVector, and both vectors are {@link #equals(FloatVector, FloatVector) equal}. + */ + @Override + boolean equals(Object obj); + + /** Returns the hash code of this vector, as defined by {@link #hash(FloatVector)}. */ + @Override + int hashCode(); + + /** + * Returns {@code true} if the given vectors are equal to each other, otherwise {@code false}. + * Two vectors are considered equal if they have the same position count, and contain the same + * values in the same order. This definition ensures that the equals method works properly + * across different implementations of the FloatVector interface. + */ + static boolean equals(FloatVector vector1, FloatVector vector2) { + final int positions = vector1.getPositionCount(); + if (positions != vector2.getPositionCount()) { + return false; + } + for (int pos = 0; pos < positions; pos++) { + if (vector1.getFloat(pos) != vector2.getFloat(pos)) { + return false; + } + } + return true; + } + + /** + * Generates the hash code for the given vector. The hash code is computed from the vector's values. + * This ensures that {@code vector1.equals(vector2)} implies that {@code vector1.hashCode()==vector2.hashCode()} + * for any two vectors, {@code vector1} and {@code vector2}, as required by the general contract of + * {@link Object#hashCode}. + */ + static int hash(FloatVector vector) { + final int len = vector.getPositionCount(); + int result = 1; + for (int pos = 0; pos < len; pos++) { + result = 31 * result + Float.floatToIntBits(vector.getFloat(pos)); + } + return result; + } + + /** Deserializes a Vector from the given stream input. */ + static FloatVector readFrom(BlockFactory blockFactory, StreamInput in) throws IOException { + final int positions = in.readVInt(); + final byte serializationType = in.readByte(); + return switch (serializationType) { + case SERIALIZE_VECTOR_VALUES -> readValues(positions, in, blockFactory); + case SERIALIZE_VECTOR_CONSTANT -> blockFactory.newConstantFloatVector(in.readFloat(), positions); + case SERIALIZE_VECTOR_ARRAY -> FloatArrayVector.readArrayVector(positions, in, blockFactory); + case SERIALIZE_VECTOR_BIG_ARRAY -> FloatBigArrayVector.readArrayVector(positions, in, blockFactory); + default -> { + assert false : "invalid vector serialization type [" + serializationType + "]"; + throw new IllegalStateException("invalid vector serialization type [" + serializationType + "]"); + } + }; + } + + /** Serializes this Vector to the given stream output. */ + default void writeTo(StreamOutput out) throws IOException { + final int positions = getPositionCount(); + final var version = out.getTransportVersion(); + out.writeVInt(positions); + if (isConstant() && positions > 0) { + out.writeByte(SERIALIZE_VECTOR_CONSTANT); + out.writeFloat(getFloat(0)); + } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_ARRAY_VECTOR) && this instanceof FloatArrayVector v) { + out.writeByte(SERIALIZE_VECTOR_ARRAY); + v.writeArrayVector(positions, out); + } else if (version.onOrAfter(TransportVersions.ESQL_SERIALIZE_BIG_VECTOR) && this instanceof FloatBigArrayVector v) { + out.writeByte(SERIALIZE_VECTOR_BIG_ARRAY); + v.writeArrayVector(positions, out); + } else { + out.writeByte(SERIALIZE_VECTOR_VALUES); + writeValues(this, positions, out); + } + } + + private static FloatVector readValues(int positions, StreamInput in, BlockFactory blockFactory) throws IOException { + try (var builder = blockFactory.newFloatVectorFixedBuilder(positions)) { + for (int i = 0; i < positions; i++) { + builder.appendFloat(i, in.readFloat()); + } + return builder.build(); + } + } + + private static void writeValues(FloatVector v, int positions, StreamOutput out) throws IOException { + for (int i = 0; i < positions; i++) { + out.writeFloat(v.getFloat(i)); + } + } + + /** + * A builder that grows as needed. + */ + sealed interface Builder extends Vector.Builder permits FloatVectorBuilder, FixedBuilder { + /** + * Appends a float to the current entry. + */ + Builder appendFloat(float value); + + @Override + FloatVector build(); + } + + /** + * A builder that never grows. + */ + sealed interface FixedBuilder extends Builder permits FloatVectorFixedBuilder { + /** + * Appends a float to the current entry. + */ + @Override + FixedBuilder appendFloat(float value); + + FixedBuilder appendFloat(int index, float value); + + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBlock.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBlock.java new file mode 100644 index 0000000000000..62a56dc6833d1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBlock.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.ReleasableIterator; +import org.elasticsearch.core.Releasables; + +/** + * Block view of a {@link FloatVector}. Cannot represent multi-values or nulls. + * This class is generated. Do not edit it. + */ +public final class FloatVectorBlock extends AbstractVectorBlock implements FloatBlock { + + private final FloatVector vector; + + /** + * @param vector considered owned by the current block; must not be used in any other {@code Block} + */ + FloatVectorBlock(FloatVector vector) { + this.vector = vector; + } + + @Override + public FloatVector asVector() { + return vector; + } + + @Override + public float getFloat(int valueIndex) { + return vector.getFloat(valueIndex); + } + + @Override + public int getPositionCount() { + return vector.getPositionCount(); + } + + @Override + public ElementType elementType() { + return vector.elementType(); + } + + @Override + public FloatBlock filter(int... positions) { + return vector.filter(positions).asBlock(); + } + + @Override + public ReleasableIterator lookup(IntBlock positions, ByteSizeValue targetBlockSize) { + return vector.lookup(positions, targetBlockSize); + } + + @Override + public FloatBlock expand() { + incRef(); + return this; + } + + @Override + public long ramBytesUsed() { + return vector.ramBytesUsed(); + } + + @Override + public boolean equals(Object obj) { + if (obj instanceof FloatBlock that) { + return FloatBlock.equals(this, that); + } + return false; + } + + @Override + public int hashCode() { + return FloatBlock.hash(this); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "[vector=" + vector + "]"; + } + + @Override + public void closeInternal() { + assert (vector.isReleased() == false) : "can't release block [" + this + "] containing already released vector"; + Releasables.closeExpectNoException(vector); + } + + @Override + public void allowPassingToDifferentDriver() { + vector.allowPassingToDifferentDriver(); + } + + @Override + public BlockFactory blockFactory() { + return vector.blockFactory(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBuilder.java new file mode 100644 index 0000000000000..9cec6355ec982 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorBuilder.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import java.util.Arrays; + +/** + * Builder for {@link FloatVector}s that grows as needed. + * This class is generated. Do not edit it. + */ +final class FloatVectorBuilder extends AbstractVectorBuilder implements FloatVector.Builder { + + private float[] values; + + FloatVectorBuilder(int estimatedSize, BlockFactory blockFactory) { + super(blockFactory); + int initialSize = Math.max(estimatedSize, 2); + adjustBreaker(initialSize); + values = new float[Math.max(estimatedSize, 2)]; + } + + @Override + public FloatVectorBuilder appendFloat(float value) { + ensureCapacity(); + values[valueCount] = value; + valueCount++; + return this; + } + + @Override + protected int elementSize() { + return Float.BYTES; + } + + @Override + protected int valuesLength() { + return values.length; + } + + @Override + protected void growValuesArray(int newSize) { + values = Arrays.copyOf(values, newSize); + } + + @Override + public FloatVector build() { + finish(); + FloatVector vector; + if (valueCount == 1) { + vector = blockFactory.newConstantFloatBlockWith(values[0], 1, estimatedBytes).asVector(); + } else { + if (values.length - valueCount > 1024 || valueCount < (values.length / 2)) { + values = Arrays.copyOf(values, valueCount); + } + vector = blockFactory.newFloatArrayVector(values, valueCount, estimatedBytes); + } + built(); + return vector; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorFixedBuilder.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorFixedBuilder.java new file mode 100644 index 0000000000000..b8d8c48823720 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/FloatVectorFixedBuilder.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.apache.lucene.util.RamUsageEstimator; + +/** + * Builder for {@link FloatVector}s that never grows. Prefer this to + * {@link FloatVectorBuilder} if you know the precise size up front because + * it's faster. + * This class is generated. Do not edit it. + */ +final class FloatVectorFixedBuilder implements FloatVector.FixedBuilder { + private final BlockFactory blockFactory; + private final float[] values; + private final long preAdjustedBytes; + /** + * The next value to write into. {@code -1} means the vector has already + * been built. + */ + private int nextIndex; + + private boolean closed; + + FloatVectorFixedBuilder(int size, BlockFactory blockFactory) { + preAdjustedBytes = ramBytesUsed(size); + blockFactory.adjustBreaker(preAdjustedBytes); + this.blockFactory = blockFactory; + this.values = new float[size]; + } + + @Override + public FloatVectorFixedBuilder appendFloat(float value) { + values[nextIndex++] = value; + return this; + } + + @Override + public FloatVectorFixedBuilder appendFloat(int idx, float value) { + values[idx] = value; + return this; + } + + private static long ramBytesUsed(int size) { + return size == 1 + ? ConstantFloatVector.RAM_BYTES_USED + : FloatArrayVector.BASE_RAM_BYTES_USED + RamUsageEstimator.alignObjectSize( + (long) RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + size * Float.BYTES + ); + } + + @Override + public long estimatedBytes() { + return ramBytesUsed(values.length); + } + + @Override + public FloatVector build() { + if (closed) { + throw new IllegalStateException("already closed"); + } + closed = true; + FloatVector vector; + if (values.length == 1) { + vector = blockFactory.newConstantFloatBlockWith(values[0], 1, preAdjustedBytes).asVector(); + } else { + vector = blockFactory.newFloatArrayVector(values, values.length, preAdjustedBytes); + } + assert vector.ramBytesUsed() == preAdjustedBytes : "fixed Builders should estimate the exact ram bytes used"; + return vector; + } + + @Override + public void close() { + if (closed == false) { + // If nextIndex < 0 we've already built the vector + closed = true; + blockFactory.adjustBreaker(-preAdjustedBytes); + } + } + + boolean isReleased() { + return closed; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/DoubleBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/DoubleBucketedSort.java new file mode 100644 index 0000000000000..63318a2189908 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/DoubleBucketedSort.java @@ -0,0 +1,346 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.DoubleArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Aggregates the top N double values per bucket. + * See {@link BucketedSort} for more information. + * This class is generated. Edit @{code X-BucketedSort.java.st} instead of this file. + */ +public class DoubleBucketedSort implements Releasable { + + private final BigArrays bigArrays; + private final SortOrder order; + private final int bucketSize; + /** + * {@code true} if the bucket is in heap mode, {@code false} if + * it is still gathering. + */ + private final BitArray heapMode; + /** + * An array containing all the values on all buckets. The structure is as follows: + *

    + * For each bucket, there are bucketSize elements, based on the bucket id (0, 1, 2...). + * Then, for each bucket, it can be in 2 states: + *

    + *
      + *
    • + * Gather mode: All buckets start in gather mode, and remain here while they have less than bucketSize elements. + * In gather mode, the elements are stored in the array from the highest index to the lowest index. + * The lowest index contains the offset to the next slot to be filled. + *

      + * This allows us to insert elements in O(1) time. + *

      + *

      + * When the bucketSize-th element is collected, the bucket transitions to heap mode, by heapifying its contents. + *

      + *
      • + *
    • + * Heap mode: The bucket slots are organized as a min heap structure. + *

      + * The root of the heap is the minimum value in the bucket, + * which allows us to quickly discard new values that are not in the top N. + *

      + *
      • + *
    + */ + private DoubleArray values; + + public DoubleBucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) { + this.bigArrays = bigArrays; + this.order = order; + this.bucketSize = bucketSize; + heapMode = new BitArray(0, bigArrays); + + boolean success = false; + try { + values = bigArrays.newDoubleArray(0, false); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + /** + * Collects a {@code value} into a {@code bucket}. + *

    + * It may or may not be inserted in the heap, depending on if it is better than the current root. + *

    + */ + public void collect(double value, int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (inHeapMode(bucket)) { + if (betterThan(value, values.get(rootIndex))) { + values.set(rootIndex, value); + downHeap(rootIndex, 0); + } + return; + } + // Gathering mode + long requiredSize = rootIndex + bucketSize; + if (values.size() < requiredSize) { + grow(requiredSize); + } + int next = getNextGatherOffset(rootIndex); + assert 0 <= next && next < bucketSize + : "Expected next to be in the range of valid buckets [0 <= " + next + " < " + bucketSize + "]"; + long index = next + rootIndex; + values.set(index, value); + if (next == 0) { + heapMode.set(bucket); + heapify(rootIndex); + } else { + setNextGatherOffset(rootIndex, next - 1); + } + } + + /** + * The order of the sort. + */ + public SortOrder getOrder() { + return order; + } + + /** + * The number of values to store per bucket. + */ + public int getBucketSize() { + return bucketSize; + } + + /** + * Get the first and last indexes (inclusive, exclusive) of the values for a bucket. + * Returns [0, 0] if the bucket has never been collected. + */ + private Tuple getBucketValuesIndexes(int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (rootIndex >= values.size()) { + // We've never seen this bucket. + return Tuple.tuple(0L, 0L); + } + long start = inHeapMode(bucket) ? rootIndex : (rootIndex + getNextGatherOffset(rootIndex) + 1); + long end = rootIndex + bucketSize; + return Tuple.tuple(start, end); + } + + /** + * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}. + */ + public void merge(int groupId, DoubleBucketedSort other, int otherGroupId) { + var otherBounds = other.getBucketValuesIndexes(otherGroupId); + + // TODO: This can be improved for heapified buckets by making use of the heap structures + for (long i = otherBounds.v1(); i < otherBounds.v2(); i++) { + collect(other.values.get(i), groupId); + } + } + + /** + * Creates a block with the values from the {@code selected} groups. + */ + public Block toBlock(BlockFactory blockFactory, IntVector selected) { + // Check if the selected groups are all empty, to avoid allocating extra memory + if (IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> { + var bounds = this.getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + return size > 0; + })) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + + // Used to sort the values in the bucket. + var bucketValues = new double[bucketSize]; + + try (var builder = blockFactory.newDoubleBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int bucket = selected.getInt(s); + + var bounds = getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + if (size == 0) { + builder.appendNull(); + continue; + } + + if (size == 1) { + builder.appendDouble(values.get(bounds.v1())); + continue; + } + + for (int i = 0; i < size; i++) { + bucketValues[i] = values.get(bounds.v1() + i); + } + + // TODO: Make use of heap structures to faster iterate in order instead of copying and sorting + Arrays.sort(bucketValues, 0, (int) size); + + builder.beginPositionEntry(); + if (order == SortOrder.ASC) { + for (int i = 0; i < size; i++) { + builder.appendDouble(bucketValues[i]); + } + } else { + for (int i = (int) size - 1; i >= 0; i--) { + builder.appendDouble(bucketValues[i]); + } + } + builder.endPositionEntry(); + } + return builder.build(); + } + } + + /** + * Is this bucket a min heap {@code true} or in gathering mode {@code false}? + */ + private boolean inHeapMode(int bucket) { + return heapMode.get(bucket); + } + + /** + * Get the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private int getNextGatherOffset(long rootIndex) { + return (int) values.get(rootIndex); + } + + /** + * Set the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private void setNextGatherOffset(long rootIndex, int offset) { + values.set(rootIndex, offset); + } + + /** + * {@code true} if the entry at index {@code lhs} is "better" than + * the entry at {@code rhs}. "Better" in this means "lower" for + * {@link SortOrder#ASC} and "higher" for {@link SortOrder#DESC}. + */ + private boolean betterThan(double lhs, double rhs) { + return getOrder().reverseMul() * Double.compare(lhs, rhs) < 0; + } + + /** + * Swap the data at two indices. + */ + private void swap(long lhs, long rhs) { + var tmp = values.get(lhs); + values.set(lhs, values.get(rhs)); + values.set(rhs, tmp); + } + + /** + * Allocate storage for more buckets and store the "next gather offset" + * for those new buckets. + */ + private void grow(long minSize) { + long oldMax = values.size(); + values = bigArrays.grow(values, minSize); + // Set the next gather offsets for all newly allocated buckets. + setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + } + + /** + * Maintain the "next gather offsets" for newly allocated buckets. + */ + private void setNextGatherOffsets(long startingAt) { + int nextOffset = getBucketSize() - 1; + for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { + setNextGatherOffset(bucketRoot, nextOffset); + } + } + + /** + * Heapify a bucket whose entries are in random order. + *

    + * This works by validating the heap property on each node, iterating + * "upwards", pushing any out of order parents "down". Check out the + * wikipedia + * entry on binary heaps for more about this. + *

    + *

    + * While this *looks* like it could easily be {@code O(n * log n)}, it is + * a fairly well studied algorithm attributed to Floyd. There's + * been a bunch of work that puts this at {@code O(n)}, close to 1.88n worst + * case. + *

    + * + * @param rootIndex the index the start of the bucket + */ + private void heapify(long rootIndex) { + int maxParent = bucketSize / 2 - 1; + for (int parent = maxParent; parent >= 0; parent--) { + downHeap(rootIndex, parent); + } + } + + /** + * Correct the heap invariant of a parent and its children. This + * runs in {@code O(log n)} time. + * @param rootIndex index of the start of the bucket + * @param parent Index within the bucket of the parent to check. + * For example, 0 is the "root". + */ + private void downHeap(long rootIndex, int parent) { + while (true) { + long parentIndex = rootIndex + parent; + int worst = parent; + long worstIndex = parentIndex; + int leftChild = parent * 2 + 1; + long leftIndex = rootIndex + leftChild; + if (leftChild < bucketSize) { + if (betterThan(values.get(worstIndex), values.get(leftIndex))) { + worst = leftChild; + worstIndex = leftIndex; + } + int rightChild = leftChild + 1; + long rightIndex = rootIndex + rightChild; + if (rightChild < bucketSize && betterThan(values.get(worstIndex), values.get(rightIndex))) { + worst = rightChild; + worstIndex = rightIndex; + } + } + if (worst == parent) { + break; + } + swap(worstIndex, parentIndex); + parent = worst; + } + } + + @Override + public final void close() { + Releasables.close(values, heapMode); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java new file mode 100644 index 0000000000000..b490fe193c33f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/FloatBucketedSort.java @@ -0,0 +1,346 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.FloatArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Aggregates the top N float values per bucket. + * See {@link BucketedSort} for more information. + * This class is generated. Edit @{code X-BucketedSort.java.st} instead of this file. + */ +public class FloatBucketedSort implements Releasable { + + private final BigArrays bigArrays; + private final SortOrder order; + private final int bucketSize; + /** + * {@code true} if the bucket is in heap mode, {@code false} if + * it is still gathering. + */ + private final BitArray heapMode; + /** + * An array containing all the values on all buckets. The structure is as follows: + *

    + * For each bucket, there are bucketSize elements, based on the bucket id (0, 1, 2...). + * Then, for each bucket, it can be in 2 states: + *

    + *
      + *
    • + * Gather mode: All buckets start in gather mode, and remain here while they have less than bucketSize elements. + * In gather mode, the elements are stored in the array from the highest index to the lowest index. + * The lowest index contains the offset to the next slot to be filled. + *

      + * This allows us to insert elements in O(1) time. + *

      + *

      + * When the bucketSize-th element is collected, the bucket transitions to heap mode, by heapifying its contents. + *

      + *
      • + *
    • + * Heap mode: The bucket slots are organized as a min heap structure. + *

      + * The root of the heap is the minimum value in the bucket, + * which allows us to quickly discard new values that are not in the top N. + *

      + *
      • + *
    + */ + private FloatArray values; + + public FloatBucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) { + this.bigArrays = bigArrays; + this.order = order; + this.bucketSize = bucketSize; + heapMode = new BitArray(0, bigArrays); + + boolean success = false; + try { + values = bigArrays.newFloatArray(0, false); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + /** + * Collects a {@code value} into a {@code bucket}. + *

    + * It may or may not be inserted in the heap, depending on if it is better than the current root. + *

    + */ + public void collect(float value, int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (inHeapMode(bucket)) { + if (betterThan(value, values.get(rootIndex))) { + values.set(rootIndex, value); + downHeap(rootIndex, 0); + } + return; + } + // Gathering mode + long requiredSize = rootIndex + bucketSize; + if (values.size() < requiredSize) { + grow(requiredSize); + } + int next = getNextGatherOffset(rootIndex); + assert 0 <= next && next < bucketSize + : "Expected next to be in the range of valid buckets [0 <= " + next + " < " + bucketSize + "]"; + long index = next + rootIndex; + values.set(index, value); + if (next == 0) { + heapMode.set(bucket); + heapify(rootIndex); + } else { + setNextGatherOffset(rootIndex, next - 1); + } + } + + /** + * The order of the sort. + */ + public SortOrder getOrder() { + return order; + } + + /** + * The number of values to store per bucket. + */ + public int getBucketSize() { + return bucketSize; + } + + /** + * Get the first and last indexes (inclusive, exclusive) of the values for a bucket. + * Returns [0, 0] if the bucket has never been collected. + */ + private Tuple getBucketValuesIndexes(int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (rootIndex >= values.size()) { + // We've never seen this bucket. + return Tuple.tuple(0L, 0L); + } + long start = inHeapMode(bucket) ? rootIndex : (rootIndex + getNextGatherOffset(rootIndex) + 1); + long end = rootIndex + bucketSize; + return Tuple.tuple(start, end); + } + + /** + * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}. + */ + public void merge(int groupId, FloatBucketedSort other, int otherGroupId) { + var otherBounds = other.getBucketValuesIndexes(otherGroupId); + + // TODO: This can be improved for heapified buckets by making use of the heap structures + for (long i = otherBounds.v1(); i < otherBounds.v2(); i++) { + collect(other.values.get(i), groupId); + } + } + + /** + * Creates a block with the values from the {@code selected} groups. + */ + public Block toBlock(BlockFactory blockFactory, IntVector selected) { + // Check if the selected groups are all empty, to avoid allocating extra memory + if (IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> { + var bounds = this.getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + return size > 0; + })) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + + // Used to sort the values in the bucket. + var bucketValues = new float[bucketSize]; + + try (var builder = blockFactory.newFloatBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int bucket = selected.getInt(s); + + var bounds = getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + if (size == 0) { + builder.appendNull(); + continue; + } + + if (size == 1) { + builder.appendFloat(values.get(bounds.v1())); + continue; + } + + for (int i = 0; i < size; i++) { + bucketValues[i] = values.get(bounds.v1() + i); + } + + // TODO: Make use of heap structures to faster iterate in order instead of copying and sorting + Arrays.sort(bucketValues, 0, (int) size); + + builder.beginPositionEntry(); + if (order == SortOrder.ASC) { + for (int i = 0; i < size; i++) { + builder.appendFloat(bucketValues[i]); + } + } else { + for (int i = (int) size - 1; i >= 0; i--) { + builder.appendFloat(bucketValues[i]); + } + } + builder.endPositionEntry(); + } + return builder.build(); + } + } + + /** + * Is this bucket a min heap {@code true} or in gathering mode {@code false}? + */ + private boolean inHeapMode(int bucket) { + return heapMode.get(bucket); + } + + /** + * Get the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private int getNextGatherOffset(long rootIndex) { + return (int) values.get(rootIndex); + } + + /** + * Set the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private void setNextGatherOffset(long rootIndex, int offset) { + values.set(rootIndex, offset); + } + + /** + * {@code true} if the entry at index {@code lhs} is "better" than + * the entry at {@code rhs}. "Better" in this means "lower" for + * {@link SortOrder#ASC} and "higher" for {@link SortOrder#DESC}. + */ + private boolean betterThan(float lhs, float rhs) { + return getOrder().reverseMul() * Float.compare(lhs, rhs) < 0; + } + + /** + * Swap the data at two indices. + */ + private void swap(long lhs, long rhs) { + var tmp = values.get(lhs); + values.set(lhs, values.get(rhs)); + values.set(rhs, tmp); + } + + /** + * Allocate storage for more buckets and store the "next gather offset" + * for those new buckets. + */ + private void grow(long minSize) { + long oldMax = values.size(); + values = bigArrays.grow(values, minSize); + // Set the next gather offsets for all newly allocated buckets. + setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + } + + /** + * Maintain the "next gather offsets" for newly allocated buckets. + */ + private void setNextGatherOffsets(long startingAt) { + int nextOffset = getBucketSize() - 1; + for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { + setNextGatherOffset(bucketRoot, nextOffset); + } + } + + /** + * Heapify a bucket whose entries are in random order. + *

    + * This works by validating the heap property on each node, iterating + * "upwards", pushing any out of order parents "down". Check out the + * wikipedia + * entry on binary heaps for more about this. + *

    + *

    + * While this *looks* like it could easily be {@code O(n * log n)}, it is + * a fairly well studied algorithm attributed to Floyd. There's + * been a bunch of work that puts this at {@code O(n)}, close to 1.88n worst + * case. + *

    + * + * @param rootIndex the index the start of the bucket + */ + private void heapify(long rootIndex) { + int maxParent = bucketSize / 2 - 1; + for (int parent = maxParent; parent >= 0; parent--) { + downHeap(rootIndex, parent); + } + } + + /** + * Correct the heap invariant of a parent and its children. This + * runs in {@code O(log n)} time. + * @param rootIndex index of the start of the bucket + * @param parent Index within the bucket of the parent to check. + * For example, 0 is the "root". + */ + private void downHeap(long rootIndex, int parent) { + while (true) { + long parentIndex = rootIndex + parent; + int worst = parent; + long worstIndex = parentIndex; + int leftChild = parent * 2 + 1; + long leftIndex = rootIndex + leftChild; + if (leftChild < bucketSize) { + if (betterThan(values.get(worstIndex), values.get(leftIndex))) { + worst = leftChild; + worstIndex = leftIndex; + } + int rightChild = leftChild + 1; + long rightIndex = rootIndex + rightChild; + if (rightChild < bucketSize && betterThan(values.get(worstIndex), values.get(rightIndex))) { + worst = rightChild; + worstIndex = rightIndex; + } + } + if (worst == parent) { + break; + } + swap(worstIndex, parentIndex); + parent = worst; + } + } + + @Override + public final void close() { + Releasables.close(values, heapMode); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/IntBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/IntBucketedSort.java new file mode 100644 index 0000000000000..04a635d75fe52 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/IntBucketedSort.java @@ -0,0 +1,346 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.IntArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Aggregates the top N int values per bucket. + * See {@link BucketedSort} for more information. + * This class is generated. Edit @{code X-BucketedSort.java.st} instead of this file. + */ +public class IntBucketedSort implements Releasable { + + private final BigArrays bigArrays; + private final SortOrder order; + private final int bucketSize; + /** + * {@code true} if the bucket is in heap mode, {@code false} if + * it is still gathering. + */ + private final BitArray heapMode; + /** + * An array containing all the values on all buckets. The structure is as follows: + *

    + * For each bucket, there are bucketSize elements, based on the bucket id (0, 1, 2...). + * Then, for each bucket, it can be in 2 states: + *

    + *
      + *
    • + * Gather mode: All buckets start in gather mode, and remain here while they have less than bucketSize elements. + * In gather mode, the elements are stored in the array from the highest index to the lowest index. + * The lowest index contains the offset to the next slot to be filled. + *

      + * This allows us to insert elements in O(1) time. + *

      + *

      + * When the bucketSize-th element is collected, the bucket transitions to heap mode, by heapifying its contents. + *

      + *
      • + *
    • + * Heap mode: The bucket slots are organized as a min heap structure. + *

      + * The root of the heap is the minimum value in the bucket, + * which allows us to quickly discard new values that are not in the top N. + *

      + *
      • + *
    + */ + private IntArray values; + + public IntBucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) { + this.bigArrays = bigArrays; + this.order = order; + this.bucketSize = bucketSize; + heapMode = new BitArray(0, bigArrays); + + boolean success = false; + try { + values = bigArrays.newIntArray(0, false); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + /** + * Collects a {@code value} into a {@code bucket}. + *

    + * It may or may not be inserted in the heap, depending on if it is better than the current root. + *

    + */ + public void collect(int value, int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (inHeapMode(bucket)) { + if (betterThan(value, values.get(rootIndex))) { + values.set(rootIndex, value); + downHeap(rootIndex, 0); + } + return; + } + // Gathering mode + long requiredSize = rootIndex + bucketSize; + if (values.size() < requiredSize) { + grow(requiredSize); + } + int next = getNextGatherOffset(rootIndex); + assert 0 <= next && next < bucketSize + : "Expected next to be in the range of valid buckets [0 <= " + next + " < " + bucketSize + "]"; + long index = next + rootIndex; + values.set(index, value); + if (next == 0) { + heapMode.set(bucket); + heapify(rootIndex); + } else { + setNextGatherOffset(rootIndex, next - 1); + } + } + + /** + * The order of the sort. + */ + public SortOrder getOrder() { + return order; + } + + /** + * The number of values to store per bucket. + */ + public int getBucketSize() { + return bucketSize; + } + + /** + * Get the first and last indexes (inclusive, exclusive) of the values for a bucket. + * Returns [0, 0] if the bucket has never been collected. + */ + private Tuple getBucketValuesIndexes(int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (rootIndex >= values.size()) { + // We've never seen this bucket. + return Tuple.tuple(0L, 0L); + } + long start = inHeapMode(bucket) ? rootIndex : (rootIndex + getNextGatherOffset(rootIndex) + 1); + long end = rootIndex + bucketSize; + return Tuple.tuple(start, end); + } + + /** + * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}. + */ + public void merge(int groupId, IntBucketedSort other, int otherGroupId) { + var otherBounds = other.getBucketValuesIndexes(otherGroupId); + + // TODO: This can be improved for heapified buckets by making use of the heap structures + for (long i = otherBounds.v1(); i < otherBounds.v2(); i++) { + collect(other.values.get(i), groupId); + } + } + + /** + * Creates a block with the values from the {@code selected} groups. + */ + public Block toBlock(BlockFactory blockFactory, IntVector selected) { + // Check if the selected groups are all empty, to avoid allocating extra memory + if (IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> { + var bounds = this.getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + return size > 0; + })) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + + // Used to sort the values in the bucket. + var bucketValues = new int[bucketSize]; + + try (var builder = blockFactory.newIntBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int bucket = selected.getInt(s); + + var bounds = getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + if (size == 0) { + builder.appendNull(); + continue; + } + + if (size == 1) { + builder.appendInt(values.get(bounds.v1())); + continue; + } + + for (int i = 0; i < size; i++) { + bucketValues[i] = values.get(bounds.v1() + i); + } + + // TODO: Make use of heap structures to faster iterate in order instead of copying and sorting + Arrays.sort(bucketValues, 0, (int) size); + + builder.beginPositionEntry(); + if (order == SortOrder.ASC) { + for (int i = 0; i < size; i++) { + builder.appendInt(bucketValues[i]); + } + } else { + for (int i = (int) size - 1; i >= 0; i--) { + builder.appendInt(bucketValues[i]); + } + } + builder.endPositionEntry(); + } + return builder.build(); + } + } + + /** + * Is this bucket a min heap {@code true} or in gathering mode {@code false}? + */ + private boolean inHeapMode(int bucket) { + return heapMode.get(bucket); + } + + /** + * Get the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private int getNextGatherOffset(long rootIndex) { + return values.get(rootIndex); + } + + /** + * Set the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private void setNextGatherOffset(long rootIndex, int offset) { + values.set(rootIndex, offset); + } + + /** + * {@code true} if the entry at index {@code lhs} is "better" than + * the entry at {@code rhs}. "Better" in this means "lower" for + * {@link SortOrder#ASC} and "higher" for {@link SortOrder#DESC}. + */ + private boolean betterThan(int lhs, int rhs) { + return getOrder().reverseMul() * Integer.compare(lhs, rhs) < 0; + } + + /** + * Swap the data at two indices. + */ + private void swap(long lhs, long rhs) { + var tmp = values.get(lhs); + values.set(lhs, values.get(rhs)); + values.set(rhs, tmp); + } + + /** + * Allocate storage for more buckets and store the "next gather offset" + * for those new buckets. + */ + private void grow(long minSize) { + long oldMax = values.size(); + values = bigArrays.grow(values, minSize); + // Set the next gather offsets for all newly allocated buckets. + setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + } + + /** + * Maintain the "next gather offsets" for newly allocated buckets. + */ + private void setNextGatherOffsets(long startingAt) { + int nextOffset = getBucketSize() - 1; + for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { + setNextGatherOffset(bucketRoot, nextOffset); + } + } + + /** + * Heapify a bucket whose entries are in random order. + *

    + * This works by validating the heap property on each node, iterating + * "upwards", pushing any out of order parents "down". Check out the + * wikipedia + * entry on binary heaps for more about this. + *

    + *

    + * While this *looks* like it could easily be {@code O(n * log n)}, it is + * a fairly well studied algorithm attributed to Floyd. There's + * been a bunch of work that puts this at {@code O(n)}, close to 1.88n worst + * case. + *

    + * + * @param rootIndex the index the start of the bucket + */ + private void heapify(long rootIndex) { + int maxParent = bucketSize / 2 - 1; + for (int parent = maxParent; parent >= 0; parent--) { + downHeap(rootIndex, parent); + } + } + + /** + * Correct the heap invariant of a parent and its children. This + * runs in {@code O(log n)} time. + * @param rootIndex index of the start of the bucket + * @param parent Index within the bucket of the parent to check. + * For example, 0 is the "root". + */ + private void downHeap(long rootIndex, int parent) { + while (true) { + long parentIndex = rootIndex + parent; + int worst = parent; + long worstIndex = parentIndex; + int leftChild = parent * 2 + 1; + long leftIndex = rootIndex + leftChild; + if (leftChild < bucketSize) { + if (betterThan(values.get(worstIndex), values.get(leftIndex))) { + worst = leftChild; + worstIndex = leftIndex; + } + int rightChild = leftChild + 1; + long rightIndex = rootIndex + rightChild; + if (rightChild < bucketSize && betterThan(values.get(worstIndex), values.get(rightIndex))) { + worst = rightChild; + worstIndex = rightIndex; + } + } + if (worst == parent) { + break; + } + swap(worstIndex, parentIndex); + parent = worst; + } + } + + @Override + public final void close() { + Releasables.close(values, heapMode); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/LongBucketedSort.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/LongBucketedSort.java new file mode 100644 index 0000000000000..e08c25256944b --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/data/sort/LongBucketedSort.java @@ -0,0 +1,346 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Aggregates the top N long values per bucket. + * See {@link BucketedSort} for more information. + * This class is generated. Edit @{code X-BucketedSort.java.st} instead of this file. + */ +public class LongBucketedSort implements Releasable { + + private final BigArrays bigArrays; + private final SortOrder order; + private final int bucketSize; + /** + * {@code true} if the bucket is in heap mode, {@code false} if + * it is still gathering. + */ + private final BitArray heapMode; + /** + * An array containing all the values on all buckets. The structure is as follows: + *

    + * For each bucket, there are bucketSize elements, based on the bucket id (0, 1, 2...). + * Then, for each bucket, it can be in 2 states: + *

    + *
      + *
    • + * Gather mode: All buckets start in gather mode, and remain here while they have less than bucketSize elements. + * In gather mode, the elements are stored in the array from the highest index to the lowest index. + * The lowest index contains the offset to the next slot to be filled. + *

      + * This allows us to insert elements in O(1) time. + *

      + *

      + * When the bucketSize-th element is collected, the bucket transitions to heap mode, by heapifying its contents. + *

      + *
      • + *
    • + * Heap mode: The bucket slots are organized as a min heap structure. + *

      + * The root of the heap is the minimum value in the bucket, + * which allows us to quickly discard new values that are not in the top N. + *

      + *
      • + *
    + */ + private LongArray values; + + public LongBucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) { + this.bigArrays = bigArrays; + this.order = order; + this.bucketSize = bucketSize; + heapMode = new BitArray(0, bigArrays); + + boolean success = false; + try { + values = bigArrays.newLongArray(0, false); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + /** + * Collects a {@code value} into a {@code bucket}. + *

    + * It may or may not be inserted in the heap, depending on if it is better than the current root. + *

    + */ + public void collect(long value, int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (inHeapMode(bucket)) { + if (betterThan(value, values.get(rootIndex))) { + values.set(rootIndex, value); + downHeap(rootIndex, 0); + } + return; + } + // Gathering mode + long requiredSize = rootIndex + bucketSize; + if (values.size() < requiredSize) { + grow(requiredSize); + } + int next = getNextGatherOffset(rootIndex); + assert 0 <= next && next < bucketSize + : "Expected next to be in the range of valid buckets [0 <= " + next + " < " + bucketSize + "]"; + long index = next + rootIndex; + values.set(index, value); + if (next == 0) { + heapMode.set(bucket); + heapify(rootIndex); + } else { + setNextGatherOffset(rootIndex, next - 1); + } + } + + /** + * The order of the sort. + */ + public SortOrder getOrder() { + return order; + } + + /** + * The number of values to store per bucket. + */ + public int getBucketSize() { + return bucketSize; + } + + /** + * Get the first and last indexes (inclusive, exclusive) of the values for a bucket. + * Returns [0, 0] if the bucket has never been collected. + */ + private Tuple getBucketValuesIndexes(int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (rootIndex >= values.size()) { + // We've never seen this bucket. + return Tuple.tuple(0L, 0L); + } + long start = inHeapMode(bucket) ? rootIndex : (rootIndex + getNextGatherOffset(rootIndex) + 1); + long end = rootIndex + bucketSize; + return Tuple.tuple(start, end); + } + + /** + * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}. + */ + public void merge(int groupId, LongBucketedSort other, int otherGroupId) { + var otherBounds = other.getBucketValuesIndexes(otherGroupId); + + // TODO: This can be improved for heapified buckets by making use of the heap structures + for (long i = otherBounds.v1(); i < otherBounds.v2(); i++) { + collect(other.values.get(i), groupId); + } + } + + /** + * Creates a block with the values from the {@code selected} groups. + */ + public Block toBlock(BlockFactory blockFactory, IntVector selected) { + // Check if the selected groups are all empty, to avoid allocating extra memory + if (IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> { + var bounds = this.getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + return size > 0; + })) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + + // Used to sort the values in the bucket. + var bucketValues = new long[bucketSize]; + + try (var builder = blockFactory.newLongBlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int bucket = selected.getInt(s); + + var bounds = getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + if (size == 0) { + builder.appendNull(); + continue; + } + + if (size == 1) { + builder.appendLong(values.get(bounds.v1())); + continue; + } + + for (int i = 0; i < size; i++) { + bucketValues[i] = values.get(bounds.v1() + i); + } + + // TODO: Make use of heap structures to faster iterate in order instead of copying and sorting + Arrays.sort(bucketValues, 0, (int) size); + + builder.beginPositionEntry(); + if (order == SortOrder.ASC) { + for (int i = 0; i < size; i++) { + builder.appendLong(bucketValues[i]); + } + } else { + for (int i = (int) size - 1; i >= 0; i--) { + builder.appendLong(bucketValues[i]); + } + } + builder.endPositionEntry(); + } + return builder.build(); + } + } + + /** + * Is this bucket a min heap {@code true} or in gathering mode {@code false}? + */ + private boolean inHeapMode(int bucket) { + return heapMode.get(bucket); + } + + /** + * Get the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private int getNextGatherOffset(long rootIndex) { + return (int) values.get(rootIndex); + } + + /** + * Set the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private void setNextGatherOffset(long rootIndex, int offset) { + values.set(rootIndex, offset); + } + + /** + * {@code true} if the entry at index {@code lhs} is "better" than + * the entry at {@code rhs}. "Better" in this means "lower" for + * {@link SortOrder#ASC} and "higher" for {@link SortOrder#DESC}. + */ + private boolean betterThan(long lhs, long rhs) { + return getOrder().reverseMul() * Long.compare(lhs, rhs) < 0; + } + + /** + * Swap the data at two indices. + */ + private void swap(long lhs, long rhs) { + var tmp = values.get(lhs); + values.set(lhs, values.get(rhs)); + values.set(rhs, tmp); + } + + /** + * Allocate storage for more buckets and store the "next gather offset" + * for those new buckets. + */ + private void grow(long minSize) { + long oldMax = values.size(); + values = bigArrays.grow(values, minSize); + // Set the next gather offsets for all newly allocated buckets. + setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + } + + /** + * Maintain the "next gather offsets" for newly allocated buckets. + */ + private void setNextGatherOffsets(long startingAt) { + int nextOffset = getBucketSize() - 1; + for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { + setNextGatherOffset(bucketRoot, nextOffset); + } + } + + /** + * Heapify a bucket whose entries are in random order. + *

    + * This works by validating the heap property on each node, iterating + * "upwards", pushing any out of order parents "down". Check out the + * wikipedia + * entry on binary heaps for more about this. + *

    + *

    + * While this *looks* like it could easily be {@code O(n * log n)}, it is + * a fairly well studied algorithm attributed to Floyd. There's + * been a bunch of work that puts this at {@code O(n)}, close to 1.88n worst + * case. + *

    + * + * @param rootIndex the index the start of the bucket + */ + private void heapify(long rootIndex) { + int maxParent = bucketSize / 2 - 1; + for (int parent = maxParent; parent >= 0; parent--) { + downHeap(rootIndex, parent); + } + } + + /** + * Correct the heap invariant of a parent and its children. This + * runs in {@code O(log n)} time. + * @param rootIndex index of the start of the bucket + * @param parent Index within the bucket of the parent to check. + * For example, 0 is the "root". + */ + private void downHeap(long rootIndex, int parent) { + while (true) { + long parentIndex = rootIndex + parent; + int worst = parent; + long worstIndex = parentIndex; + int leftChild = parent * 2 + 1; + long leftIndex = rootIndex + leftChild; + if (leftChild < bucketSize) { + if (betterThan(values.get(worstIndex), values.get(leftIndex))) { + worst = leftChild; + worstIndex = leftIndex; + } + int rightChild = leftChild + 1; + long rightIndex = rootIndex + rightChild; + if (rightChild < bucketSize && betterThan(values.get(worstIndex), values.get(rightIndex))) { + worst = rightChild; + worstIndex = rightIndex; + } + } + if (worst == parent) { + break; + } + swap(worstIndex, parentIndex); + parent = worst; + } + } + + @Override + public final void close() { + Releasables.close(values, heapMode); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForFloat.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForFloat.java new file mode 100644 index 0000000000000..66cd1c88f5067 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/KeyExtractorForFloat.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator.topn; + +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; + +import java.util.Locale; + +/** + * Extracts sort keys for top-n from their {@link FloatBlock}s. + * This class is generated. Edit {@code X-KeyExtractor.java.st} instead. + */ +abstract class KeyExtractorForFloat implements KeyExtractor { + static KeyExtractorForFloat extractorFor(TopNEncoder encoder, boolean ascending, byte nul, byte nonNul, FloatBlock block) { + FloatVector v = block.asVector(); + if (v != null) { + return new KeyExtractorForFloat.FromVector(encoder, nul, nonNul, v); + } + if (ascending) { + return block.mvSortedAscending() + ? new KeyExtractorForFloat.MinFromAscendingBlock(encoder, nul, nonNul, block) + : new KeyExtractorForFloat.MinFromUnorderedBlock(encoder, nul, nonNul, block); + } + return block.mvSortedAscending() + ? new KeyExtractorForFloat.MaxFromAscendingBlock(encoder, nul, nonNul, block) + : new KeyExtractorForFloat.MaxFromUnorderedBlock(encoder, nul, nonNul, block); + } + + private final byte nul; + private final byte nonNul; + + KeyExtractorForFloat(TopNEncoder encoder, byte nul, byte nonNul) { + assert encoder == TopNEncoder.DEFAULT_SORTABLE; + this.nul = nul; + this.nonNul = nonNul; + } + + protected final int nonNul(BreakingBytesRefBuilder key, float value) { + key.append(nonNul); + TopNEncoder.DEFAULT_SORTABLE.encodeFloat(value, key); + return Float.BYTES + 1; + } + + protected final int nul(BreakingBytesRefBuilder key) { + key.append(nul); + return 1; + } + + @Override + public final String toString() { + return String.format(Locale.ROOT, "KeyExtractorForFloat%s(%s, %s)", getClass().getSimpleName(), nul, nonNul); + } + + static class FromVector extends KeyExtractorForFloat { + private final FloatVector vector; + + FromVector(TopNEncoder encoder, byte nul, byte nonNul, FloatVector vector) { + super(encoder, nul, nonNul); + this.vector = vector; + } + + @Override + public int writeKey(BreakingBytesRefBuilder key, int position) { + return nonNul(key, vector.getFloat(position)); + } + } + + static class MinFromAscendingBlock extends KeyExtractorForFloat { + private final FloatBlock block; + + MinFromAscendingBlock(TopNEncoder encoder, byte nul, byte nonNul, FloatBlock block) { + super(encoder, nul, nonNul); + this.block = block; + } + + @Override + public int writeKey(BreakingBytesRefBuilder key, int position) { + if (block.isNull(position)) { + return nul(key); + } + return nonNul(key, block.getFloat(block.getFirstValueIndex(position))); + } + } + + static class MaxFromAscendingBlock extends KeyExtractorForFloat { + private final FloatBlock block; + + MaxFromAscendingBlock(TopNEncoder encoder, byte nul, byte nonNul, FloatBlock block) { + super(encoder, nul, nonNul); + this.block = block; + } + + @Override + public int writeKey(BreakingBytesRefBuilder key, int position) { + if (block.isNull(position)) { + return nul(key); + } + return nonNul(key, block.getFloat(block.getFirstValueIndex(position) + block.getValueCount(position) - 1)); + } + } + + static class MinFromUnorderedBlock extends KeyExtractorForFloat { + private final FloatBlock block; + + MinFromUnorderedBlock(TopNEncoder encoder, byte nul, byte nonNul, FloatBlock block) { + super(encoder, nul, nonNul); + this.block = block; + } + + @Override + public int writeKey(BreakingBytesRefBuilder key, int position) { + int size = block.getValueCount(position); + if (size == 0) { + return nul(key); + } + int start = block.getFirstValueIndex(position); + int end = start + size; + float min = block.getFloat(start); + for (int i = start + 1; i < end; i++) { + min = Math.min(min, block.getFloat(i)); + } + return nonNul(key, min); + } + } + + static class MaxFromUnorderedBlock extends KeyExtractorForFloat { + private final FloatBlock block; + + MaxFromUnorderedBlock(TopNEncoder encoder, byte nul, byte nonNul, FloatBlock block) { + super(encoder, nul, nonNul); + this.block = block; + } + + @Override + public int writeKey(BreakingBytesRefBuilder key, int position) { + int size = block.getValueCount(position); + if (size == 0) { + return nul(key); + } + int start = block.getFirstValueIndex(position); + int end = start + size; + float max = block.getFloat(start); + for (int i = start + 1; i < end; i++) { + max = Math.max(max, block.getFloat(i)); + } + return nonNul(key, max); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ResultBuilderForFloat.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ResultBuilderForFloat.java new file mode 100644 index 0000000000000..a417f1c0b77d1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ResultBuilderForFloat.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator.topn; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; + +/** + * Builds the resulting {@link FloatBlock} for some column in a top-n. + * This class is generated. Edit {@code X-ResultBuilder.java.st} instead. + */ +class ResultBuilderForFloat implements ResultBuilder { + private final FloatBlock.Builder builder; + + private final boolean inKey; + + /** + * The value previously set by {@link #decodeKey}. + */ + private float key; + + ResultBuilderForFloat(BlockFactory blockFactory, TopNEncoder encoder, boolean inKey, int initialSize) { + assert encoder == TopNEncoder.DEFAULT_UNSORTABLE : encoder.toString(); + this.inKey = inKey; + this.builder = blockFactory.newFloatBlockBuilder(initialSize); + } + + @Override + public void decodeKey(BytesRef keys) { + assert inKey; + key = TopNEncoder.DEFAULT_SORTABLE.decodeFloat(keys); + } + + @Override + public void decodeValue(BytesRef values) { + int count = TopNEncoder.DEFAULT_UNSORTABLE.decodeVInt(values); + switch (count) { + case 0 -> { + builder.appendNull(); + } + case 1 -> builder.appendFloat(inKey ? key : readValueFromValues(values)); + default -> { + builder.beginPositionEntry(); + for (int i = 0; i < count; i++) { + builder.appendFloat(readValueFromValues(values)); + } + builder.endPositionEntry(); + } + } + } + + private float readValueFromValues(BytesRef values) { + return TopNEncoder.DEFAULT_UNSORTABLE.decodeFloat(values); + } + + @Override + public FloatBlock build() { + return builder.build(); + } + + @Override + public String toString() { + return "ResultBuilderForFloat[inKey=" + inKey + "]"; + } + + @Override + public void close() { + builder.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForFloat.java b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForFloat.java new file mode 100644 index 0000000000000..295ef755a2225 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated-src/org/elasticsearch/compute/operator/topn/ValueExtractorForFloat.java @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator.topn; + +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; + +/** + * Extracts non-sort-key values for top-n from their {@link FloatBlock}s. + * This class is generated. Edit {@code X-KeyExtractor.java.st} instead. + */ +abstract class ValueExtractorForFloat implements ValueExtractor { + static ValueExtractorForFloat extractorFor(TopNEncoder encoder, boolean inKey, FloatBlock block) { + FloatVector vector = block.asVector(); + if (vector != null) { + return new ValueExtractorForFloat.ForVector(encoder, inKey, vector); + } + return new ValueExtractorForFloat.ForBlock(encoder, inKey, block); + } + + protected final boolean inKey; + + ValueExtractorForFloat(TopNEncoder encoder, boolean inKey) { + assert encoder == TopNEncoder.DEFAULT_UNSORTABLE : encoder.toString(); + this.inKey = inKey; + } + + protected final void writeCount(BreakingBytesRefBuilder values, int count) { + TopNEncoder.DEFAULT_UNSORTABLE.encodeVInt(count, values); + } + + protected final void actualWriteValue(BreakingBytesRefBuilder values, float value) { + TopNEncoder.DEFAULT_UNSORTABLE.encodeFloat(value, values); + } + + static class ForVector extends ValueExtractorForFloat { + private final FloatVector vector; + + ForVector(TopNEncoder encoder, boolean inKey, FloatVector vector) { + super(encoder, inKey); + this.vector = vector; + } + + @Override + public void writeValue(BreakingBytesRefBuilder values, int position) { + writeCount(values, 1); + if (inKey) { + // will read results from the key + return; + } + actualWriteValue(values, vector.getFloat(position)); + } + } + + static class ForBlock extends ValueExtractorForFloat { + private final FloatBlock block; + + ForBlock(TopNEncoder encoder, boolean inKey, FloatBlock block) { + super(encoder, inKey); + this.block = block; + } + + @Override + public void writeValue(BreakingBytesRefBuilder values, int position) { + int size = block.getValueCount(position); + writeCount(values, size); + if (size == 1 && inKey) { + // Will read results from the key + return; + } + int start = block.getFirstValueIndex(position); + int end = start + size; + for (int i = start; i < end; i++) { + actualWriteValue(values, block.getFloat(i)); + } + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java new file mode 100644 index 0000000000000..aad616eac95a1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunction.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link CountDistinctFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class CountDistinctFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("hll", ElementType.BYTES_REF) ); + + private final DriverContext driverContext; + + private final HllStates.SingleState state; + + private final List channels; + + private final int precision; + + public CountDistinctFloatAggregatorFunction(DriverContext driverContext, List channels, + HllStates.SingleState state, int precision) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.precision = precision; + } + + public static CountDistinctFloatAggregatorFunction create(DriverContext driverContext, + List channels, int precision) { + return new CountDistinctFloatAggregatorFunction(driverContext, channels, CountDistinctFloatAggregator.initSingle(driverContext.bigArrays(), precision), precision); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + CountDistinctFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + CountDistinctFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block hllUncast = page.getBlock(channels.get(0)); + if (hllUncast.areAllValuesNull()) { + return; + } + BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); + assert hll.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + CountDistinctFloatAggregator.combineIntermediate(state, hll.getBytesRef(0, scratch)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = CountDistinctFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..4c2aad00a7a72 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionSupplier.java @@ -0,0 +1,42 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link CountDistinctFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class CountDistinctFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int precision; + + public CountDistinctFloatAggregatorFunctionSupplier(List channels, int precision) { + this.channels = channels; + this.precision = precision; + } + + @Override + public CountDistinctFloatAggregatorFunction aggregator(DriverContext driverContext) { + return CountDistinctFloatAggregatorFunction.create(driverContext, channels, precision); + } + + @Override + public CountDistinctFloatGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return CountDistinctFloatGroupingAggregatorFunction.create(channels, driverContext, precision); + } + + @Override + public String describe() { + return "count_distinct of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..60c1755b88c6a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link CountDistinctFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class CountDistinctFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("hll", ElementType.BYTES_REF) ); + + private final HllStates.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int precision; + + public CountDistinctFloatGroupingAggregatorFunction(List channels, + HllStates.GroupingState state, DriverContext driverContext, int precision) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.precision = precision; + } + + public static CountDistinctFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int precision) { + return new CountDistinctFloatGroupingAggregatorFunction(channels, CountDistinctFloatAggregator.initGrouping(driverContext.bigArrays(), precision), driverContext, precision); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + CountDistinctFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block hllUncast = page.getBlock(channels.get(0)); + if (hllUncast.areAllValuesNull()) { + return; + } + BytesRefVector hll = ((BytesRefBlock) hllUncast).asVector(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + CountDistinctFloatAggregator.combineIntermediate(state, groupId, hll.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + HllStates.GroupingState inState = ((CountDistinctFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + CountDistinctFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = CountDistinctFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java new file mode 100644 index 0000000000000..0dcef4341727d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunction.java @@ -0,0 +1,138 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MaxFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.FLOAT), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final FloatState state; + + private final List channels; + + public MaxFloatAggregatorFunction(DriverContext driverContext, List channels, + FloatState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MaxFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MaxFloatAggregatorFunction(driverContext, channels, new FloatState(MaxFloatAggregator.init())); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + state.seen(true); + for (int i = 0; i < vector.getPositionCount(); i++) { + state.floatValue(MaxFloatAggregator.combine(state.floatValue(), vector.getFloat(i))); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + state.seen(true); + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + state.floatValue(MaxFloatAggregator.combine(state.floatValue(), block.getFloat(i))); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + FloatVector max = ((FloatBlock) maxUncast).asVector(); + assert max.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + if (seen.getBoolean(0)) { + state.floatValue(MaxFloatAggregator.combine(state.floatValue(), max.getFloat(0))); + state.seen(true); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + if (state.seen() == false) { + blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1); + return; + } + blocks[offset] = driverContext.blockFactory().newConstantFloatBlockWith(state.floatValue(), 1); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..a3aa44f432430 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MaxFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MaxFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MaxFloatAggregatorFunction aggregator(DriverContext driverContext) { + return MaxFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public MaxFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return MaxFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "max of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..85708792732a7 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunction.java @@ -0,0 +1,208 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MaxFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MaxFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("max", ElementType.FLOAT), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final FloatArrayState state; + + private final List channels; + + private final DriverContext driverContext; + + public MaxFloatGroupingAggregatorFunction(List channels, FloatArrayState state, + DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MaxFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new MaxFloatGroupingAggregatorFunction(channels, new FloatArrayState(driverContext.bigArrays(), MaxFloatAggregator.init()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(v))); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset))); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(v))); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block maxUncast = page.getBlock(channels.get(0)); + if (maxUncast.areAllValuesNull()) { + return; + } + FloatVector max = ((FloatBlock) maxUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert max.getPositionCount() == seen.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (seen.getBoolean(groupPosition + positionOffset)) { + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), max.getFloat(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + FloatArrayState inState = ((MaxFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + if (inState.hasValue(position)) { + state.set(groupId, MaxFloatAggregator.combine(state.getOrDefault(groupId), inState.get(position))); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = state.toValuesBlock(selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java new file mode 100644 index 0000000000000..38a16859140f8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunction.java @@ -0,0 +1,124 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MedianAbsoluteDeviationFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MedianAbsoluteDeviationFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("quart", ElementType.BYTES_REF) ); + + private final DriverContext driverContext; + + private final QuantileStates.SingleState state; + + private final List channels; + + public MedianAbsoluteDeviationFloatAggregatorFunction(DriverContext driverContext, + List channels, QuantileStates.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MedianAbsoluteDeviationFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MedianAbsoluteDeviationFloatAggregatorFunction(driverContext, channels, MedianAbsoluteDeviationFloatAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + MedianAbsoluteDeviationFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + MedianAbsoluteDeviationFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block quartUncast = page.getBlock(channels.get(0)); + if (quartUncast.areAllValuesNull()) { + return; + } + BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); + assert quart.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = MedianAbsoluteDeviationFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..1fad0faafad4e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionSupplier.java @@ -0,0 +1,39 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MedianAbsoluteDeviationFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MedianAbsoluteDeviationFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MedianAbsoluteDeviationFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MedianAbsoluteDeviationFloatAggregatorFunction aggregator(DriverContext driverContext) { + return MedianAbsoluteDeviationFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public MedianAbsoluteDeviationFloatGroupingAggregatorFunction groupingAggregator( + DriverContext driverContext) { + return MedianAbsoluteDeviationFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "median_absolute_deviation of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..84646476fcee0 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunction.java @@ -0,0 +1,199 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MedianAbsoluteDeviationFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MedianAbsoluteDeviationFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("quart", ElementType.BYTES_REF) ); + + private final QuantileStates.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public MedianAbsoluteDeviationFloatGroupingAggregatorFunction(List channels, + QuantileStates.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MedianAbsoluteDeviationFloatGroupingAggregatorFunction create( + List channels, DriverContext driverContext) { + return new MedianAbsoluteDeviationFloatGroupingAggregatorFunction(channels, MedianAbsoluteDeviationFloatAggregator.initGrouping(driverContext.bigArrays()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + MedianAbsoluteDeviationFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block quartUncast = page.getBlock(channels.get(0)); + if (quartUncast.areAllValuesNull()) { + return; + } + BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + MedianAbsoluteDeviationFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + QuantileStates.GroupingState inState = ((MedianAbsoluteDeviationFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + MedianAbsoluteDeviationFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = MedianAbsoluteDeviationFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java new file mode 100644 index 0000000000000..ecabcbdcf57bb --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunction.java @@ -0,0 +1,138 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link MinFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("min", ElementType.FLOAT), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final FloatState state; + + private final List channels; + + public MinFloatAggregatorFunction(DriverContext driverContext, List channels, + FloatState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static MinFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new MinFloatAggregatorFunction(driverContext, channels, new FloatState(MinFloatAggregator.init())); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + state.seen(true); + for (int i = 0; i < vector.getPositionCount(); i++) { + state.floatValue(MinFloatAggregator.combine(state.floatValue(), vector.getFloat(i))); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + state.seen(true); + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + state.floatValue(MinFloatAggregator.combine(state.floatValue(), block.getFloat(i))); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block minUncast = page.getBlock(channels.get(0)); + if (minUncast.areAllValuesNull()) { + return; + } + FloatVector min = ((FloatBlock) minUncast).asVector(); + assert min.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + if (seen.getBoolean(0)) { + state.floatValue(MinFloatAggregator.combine(state.floatValue(), min.getFloat(0))); + state.seen(true); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + if (state.seen() == false) { + blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1); + return; + } + blocks[offset] = driverContext.blockFactory().newConstantFloatBlockWith(state.floatValue(), 1); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..a8ccc70f9996a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link MinFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public MinFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public MinFloatAggregatorFunction aggregator(DriverContext driverContext) { + return MinFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public MinFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return MinFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "min of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..2f00bbf1335ed --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunction.java @@ -0,0 +1,208 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link MinFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class MinFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("min", ElementType.FLOAT), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final FloatArrayState state; + + private final List channels; + + private final DriverContext driverContext; + + public MinFloatGroupingAggregatorFunction(List channels, FloatArrayState state, + DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static MinFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new MinFloatGroupingAggregatorFunction(channels, new FloatArrayState(driverContext.bigArrays(), MinFloatAggregator.init()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(v))); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset))); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(v))); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), values.getFloat(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block minUncast = page.getBlock(channels.get(0)); + if (minUncast.areAllValuesNull()) { + return; + } + FloatVector min = ((FloatBlock) minUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(1)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert min.getPositionCount() == seen.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (seen.getBoolean(groupPosition + positionOffset)) { + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), min.getFloat(groupPosition + positionOffset))); + } + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + FloatArrayState inState = ((MinFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + if (inState.hasValue(position)) { + state.set(groupId, MinFloatAggregator.combine(state.getOrDefault(groupId), inState.get(position))); + } + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = state.toValuesBlock(selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java new file mode 100644 index 0000000000000..8f0ffd81e64b6 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunction.java @@ -0,0 +1,127 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link PercentileFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class PercentileFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("quart", ElementType.BYTES_REF) ); + + private final DriverContext driverContext; + + private final QuantileStates.SingleState state; + + private final List channels; + + private final double percentile; + + public PercentileFloatAggregatorFunction(DriverContext driverContext, List channels, + QuantileStates.SingleState state, double percentile) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.percentile = percentile; + } + + public static PercentileFloatAggregatorFunction create(DriverContext driverContext, + List channels, double percentile) { + return new PercentileFloatAggregatorFunction(driverContext, channels, PercentileFloatAggregator.initSingle(percentile), percentile); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + PercentileFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + PercentileFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block quartUncast = page.getBlock(channels.get(0)); + if (quartUncast.areAllValuesNull()) { + return; + } + BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); + assert quart.getPositionCount() == 1; + BytesRef scratch = new BytesRef(); + PercentileFloatAggregator.combineIntermediate(state, quart.getBytesRef(0, scratch)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = PercentileFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..1d1678f15448c --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionSupplier.java @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link PercentileFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class PercentileFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final double percentile; + + public PercentileFloatAggregatorFunctionSupplier(List channels, double percentile) { + this.channels = channels; + this.percentile = percentile; + } + + @Override + public PercentileFloatAggregatorFunction aggregator(DriverContext driverContext) { + return PercentileFloatAggregatorFunction.create(driverContext, channels, percentile); + } + + @Override + public PercentileFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return PercentileFloatGroupingAggregatorFunction.create(channels, driverContext, percentile); + } + + @Override + public String describe() { + return "percentile of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..564e0e90018c2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link PercentileFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class PercentileFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("quart", ElementType.BYTES_REF) ); + + private final QuantileStates.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final double percentile; + + public PercentileFloatGroupingAggregatorFunction(List channels, + QuantileStates.GroupingState state, DriverContext driverContext, double percentile) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.percentile = percentile; + } + + public static PercentileFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext, double percentile) { + return new PercentileFloatGroupingAggregatorFunction(channels, PercentileFloatAggregator.initGrouping(driverContext.bigArrays(), percentile), driverContext, percentile); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + PercentileFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + PercentileFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + PercentileFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + PercentileFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block quartUncast = page.getBlock(channels.get(0)); + if (quartUncast.areAllValuesNull()) { + return; + } + BytesRefVector quart = ((BytesRefBlock) quartUncast).asVector(); + BytesRef scratch = new BytesRef(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + PercentileFloatAggregator.combineIntermediate(state, groupId, quart.getBytesRef(groupPosition + positionOffset, scratch)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + QuantileStates.GroupingState inState = ((PercentileFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + PercentileFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = PercentileFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..4b1546314a9cb --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatAggregatorFunctionSupplier.java @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link RateFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class RateFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final long unitInMillis; + + public RateFloatAggregatorFunctionSupplier(List channels, long unitInMillis) { + this.channels = channels; + this.unitInMillis = unitInMillis; + } + + @Override + public AggregatorFunction aggregator(DriverContext driverContext) { + throw new UnsupportedOperationException("non-grouping aggregator is not supported"); + } + + @Override + public RateFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return RateFloatGroupingAggregatorFunction.create(channels, driverContext, unitInMillis); + } + + @Override + public String describe() { + return "rate of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..40f53741bf3da --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/RateFloatGroupingAggregatorFunction.java @@ -0,0 +1,227 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link RateFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class RateFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("timestamps", ElementType.LONG), + new IntermediateStateDesc("values", ElementType.FLOAT), + new IntermediateStateDesc("resets", ElementType.DOUBLE) ); + + private final RateFloatAggregator.FloatRateGroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final long unitInMillis; + + public RateFloatGroupingAggregatorFunction(List channels, + RateFloatAggregator.FloatRateGroupingState state, DriverContext driverContext, + long unitInMillis) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.unitInMillis = unitInMillis; + } + + public static RateFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext, long unitInMillis) { + return new RateFloatGroupingAggregatorFunction(channels, RateFloatAggregator.initGrouping(driverContext, unitInMillis), driverContext, unitInMillis); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + LongBlock timestampsBlock = page.getBlock(channels.get(1)); + LongVector timestampsVector = timestampsBlock.asVector(); + if (timestampsVector == null) { + throw new IllegalStateException("expected @timestamp vector; but got a block"); + } + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock, timestampsVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock, timestampsVector); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector, timestampsVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector, timestampsVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + RateFloatAggregator.combine(state, groupId, timestamps.getLong(v), values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + var valuePosition = groupPosition + positionOffset; + RateFloatAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getFloat(valuePosition)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + RateFloatAggregator.combine(state, groupId, timestamps.getLong(v), values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values, + LongVector timestamps) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + var valuePosition = groupPosition + positionOffset; + RateFloatAggregator.combine(state, groupId, timestamps.getLong(valuePosition), values.getFloat(valuePosition)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block timestampsUncast = page.getBlock(channels.get(0)); + if (timestampsUncast.areAllValuesNull()) { + return; + } + LongBlock timestamps = (LongBlock) timestampsUncast; + Block valuesUncast = page.getBlock(channels.get(1)); + if (valuesUncast.areAllValuesNull()) { + return; + } + FloatBlock values = (FloatBlock) valuesUncast; + Block resetsUncast = page.getBlock(channels.get(2)); + if (resetsUncast.areAllValuesNull()) { + return; + } + DoubleVector resets = ((DoubleBlock) resetsUncast).asVector(); + assert timestamps.getPositionCount() == values.getPositionCount() && timestamps.getPositionCount() == resets.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + RateFloatAggregator.combineIntermediate(state, groupId, timestamps, values, resets.getDouble(groupPosition + positionOffset), groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + RateFloatAggregator.FloatRateGroupingState inState = ((RateFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + RateFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = RateFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java new file mode 100644 index 0000000000000..3dedc327294d5 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunction.java @@ -0,0 +1,144 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link SumFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class SumFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("value", ElementType.DOUBLE), + new IntermediateStateDesc("delta", ElementType.DOUBLE), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final DriverContext driverContext; + + private final SumDoubleAggregator.SumState state; + + private final List channels; + + public SumFloatAggregatorFunction(DriverContext driverContext, List channels, + SumDoubleAggregator.SumState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static SumFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new SumFloatAggregatorFunction(driverContext, channels, SumFloatAggregator.initSingle()); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + state.seen(true); + for (int i = 0; i < vector.getPositionCount(); i++) { + SumFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + state.seen(true); + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + SumFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block valueUncast = page.getBlock(channels.get(0)); + if (valueUncast.areAllValuesNull()) { + return; + } + DoubleVector value = ((DoubleBlock) valueUncast).asVector(); + assert value.getPositionCount() == 1; + Block deltaUncast = page.getBlock(channels.get(1)); + if (deltaUncast.areAllValuesNull()) { + return; + } + DoubleVector delta = ((DoubleBlock) deltaUncast).asVector(); + assert delta.getPositionCount() == 1; + Block seenUncast = page.getBlock(channels.get(2)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert seen.getPositionCount() == 1; + SumFloatAggregator.combineIntermediate(state, value.getDouble(0), delta.getDouble(0), seen.getBoolean(0)); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + if (state.seen() == false) { + blocks[offset] = driverContext.blockFactory().newConstantNullBlock(1); + return; + } + blocks[offset] = SumFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..515122ec08ac0 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link SumFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class SumFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public SumFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public SumFloatAggregatorFunction aggregator(DriverContext driverContext) { + return SumFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public SumFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return SumFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "sum of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..c69ce16f0bccb --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunction.java @@ -0,0 +1,212 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BooleanVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link SumFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class SumFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("value", ElementType.DOUBLE), + new IntermediateStateDesc("delta", ElementType.DOUBLE), + new IntermediateStateDesc("seen", ElementType.BOOLEAN) ); + + private final SumDoubleAggregator.GroupingSumState state; + + private final List channels; + + private final DriverContext driverContext; + + public SumFloatGroupingAggregatorFunction(List channels, + SumDoubleAggregator.GroupingSumState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static SumFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new SumFloatGroupingAggregatorFunction(channels, SumFloatAggregator.initGrouping(driverContext.bigArrays()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + SumFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + SumFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + SumFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + SumFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block valueUncast = page.getBlock(channels.get(0)); + if (valueUncast.areAllValuesNull()) { + return; + } + DoubleVector value = ((DoubleBlock) valueUncast).asVector(); + Block deltaUncast = page.getBlock(channels.get(1)); + if (deltaUncast.areAllValuesNull()) { + return; + } + DoubleVector delta = ((DoubleBlock) deltaUncast).asVector(); + Block seenUncast = page.getBlock(channels.get(2)); + if (seenUncast.areAllValuesNull()) { + return; + } + BooleanVector seen = ((BooleanBlock) seenUncast).asVector(); + assert value.getPositionCount() == delta.getPositionCount() && value.getPositionCount() == seen.getPositionCount(); + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + SumFloatAggregator.combineIntermediate(state, groupId, value.getDouble(groupPosition + positionOffset), delta.getDouble(groupPosition + positionOffset), seen.getBoolean(groupPosition + positionOffset)); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + SumDoubleAggregator.GroupingSumState inState = ((SumFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + SumFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = SumFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunction.java new file mode 100644 index 0000000000000..d52d25941780c --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunction.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link TopListDoubleAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListDoubleAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.DOUBLE) ); + + private final DriverContext driverContext; + + private final TopListDoubleAggregator.SingleState state; + + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListDoubleAggregatorFunction(DriverContext driverContext, List channels, + TopListDoubleAggregator.SingleState state, int limit, boolean ascending) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListDoubleAggregatorFunction create(DriverContext driverContext, + List channels, int limit, boolean ascending) { + return new TopListDoubleAggregatorFunction(driverContext, channels, TopListDoubleAggregator.initSingle(driverContext.bigArrays(), limit, ascending), limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + DoubleBlock block = page.getBlock(channels.get(0)); + DoubleVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(DoubleVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + TopListDoubleAggregator.combine(state, vector.getDouble(i)); + } + } + + private void addRawBlock(DoubleBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + TopListDoubleAggregator.combine(state, block.getDouble(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + DoubleBlock topList = (DoubleBlock) topListUncast; + assert topList.getPositionCount() == 1; + TopListDoubleAggregator.combineIntermediate(state, topList); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = TopListDoubleAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..48df091d339b6 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunctionSupplier.java @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link TopListDoubleAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListDoubleAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListDoubleAggregatorFunctionSupplier(List channels, int limit, + boolean ascending) { + this.channels = channels; + this.limit = limit; + this.ascending = ascending; + } + + @Override + public TopListDoubleAggregatorFunction aggregator(DriverContext driverContext) { + return TopListDoubleAggregatorFunction.create(driverContext, channels, limit, ascending); + } + + @Override + public TopListDoubleGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return TopListDoubleGroupingAggregatorFunction.create(channels, driverContext, limit, ascending); + } + + @Override + public String describe() { + return "top_list of doubles"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..0e3b98bb0f7e5 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListDoubleGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link TopListDoubleAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListDoubleGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.DOUBLE) ); + + private final TopListDoubleAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int limit; + + private final boolean ascending; + + public TopListDoubleGroupingAggregatorFunction(List channels, + TopListDoubleAggregator.GroupingState state, DriverContext driverContext, int limit, + boolean ascending) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListDoubleGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int limit, boolean ascending) { + return new TopListDoubleGroupingAggregatorFunction(channels, TopListDoubleAggregator.initGrouping(driverContext.bigArrays(), limit, ascending), driverContext, limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + DoubleBlock valuesBlock = page.getBlock(channels.get(0)); + DoubleVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, DoubleBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListDoubleAggregator.combine(state, groupId, values.getDouble(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, DoubleVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, DoubleBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListDoubleAggregator.combine(state, groupId, values.getDouble(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, DoubleVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + TopListDoubleAggregator.combine(state, groupId, values.getDouble(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + DoubleBlock topList = (DoubleBlock) topListUncast; + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListDoubleAggregator.combineIntermediate(state, groupId, topList, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + TopListDoubleAggregator.GroupingState inState = ((TopListDoubleGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + TopListDoubleAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = TopListDoubleAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunction.java new file mode 100644 index 0000000000000..6232d6ff21fc9 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunction.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link TopListFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.FLOAT) ); + + private final DriverContext driverContext; + + private final TopListFloatAggregator.SingleState state; + + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListFloatAggregatorFunction(DriverContext driverContext, List channels, + TopListFloatAggregator.SingleState state, int limit, boolean ascending) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListFloatAggregatorFunction create(DriverContext driverContext, + List channels, int limit, boolean ascending) { + return new TopListFloatAggregatorFunction(driverContext, channels, TopListFloatAggregator.initSingle(driverContext.bigArrays(), limit, ascending), limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + TopListFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + TopListFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + FloatBlock topList = (FloatBlock) topListUncast; + assert topList.getPositionCount() == 1; + TopListFloatAggregator.combineIntermediate(state, topList); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = TopListFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..ff1c3e8df4b46 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionSupplier.java @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link TopListFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListFloatAggregatorFunctionSupplier(List channels, int limit, + boolean ascending) { + this.channels = channels; + this.limit = limit; + this.ascending = ascending; + } + + @Override + public TopListFloatAggregatorFunction aggregator(DriverContext driverContext) { + return TopListFloatAggregatorFunction.create(driverContext, channels, limit, ascending); + } + + @Override + public TopListFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return TopListFloatGroupingAggregatorFunction.create(channels, driverContext, limit, ascending); + } + + @Override + public String describe() { + return "top_list of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..66f8fa7eeb35d --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListFloatGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link TopListFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.FLOAT) ); + + private final TopListFloatAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int limit; + + private final boolean ascending; + + public TopListFloatGroupingAggregatorFunction(List channels, + TopListFloatAggregator.GroupingState state, DriverContext driverContext, int limit, + boolean ascending) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int limit, boolean ascending) { + return new TopListFloatGroupingAggregatorFunction(channels, TopListFloatAggregator.initGrouping(driverContext.bigArrays(), limit, ascending), driverContext, limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + TopListFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + FloatBlock topList = (FloatBlock) topListUncast; + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListFloatAggregator.combineIntermediate(state, groupId, topList, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + TopListFloatAggregator.GroupingState inState = ((TopListFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + TopListFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = TopListFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunction.java new file mode 100644 index 0000000000000..e885b285c4a51 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunction.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link TopListIntAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListIntAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.INT) ); + + private final DriverContext driverContext; + + private final TopListIntAggregator.SingleState state; + + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListIntAggregatorFunction(DriverContext driverContext, List channels, + TopListIntAggregator.SingleState state, int limit, boolean ascending) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListIntAggregatorFunction create(DriverContext driverContext, + List channels, int limit, boolean ascending) { + return new TopListIntAggregatorFunction(driverContext, channels, TopListIntAggregator.initSingle(driverContext.bigArrays(), limit, ascending), limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + IntBlock block = page.getBlock(channels.get(0)); + IntVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(IntVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + TopListIntAggregator.combine(state, vector.getInt(i)); + } + } + + private void addRawBlock(IntBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + TopListIntAggregator.combine(state, block.getInt(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + IntBlock topList = (IntBlock) topListUncast; + assert topList.getPositionCount() == 1; + TopListIntAggregator.combineIntermediate(state, topList); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = TopListIntAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..d8bf91ba85541 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunctionSupplier.java @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link TopListIntAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListIntAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListIntAggregatorFunctionSupplier(List channels, int limit, + boolean ascending) { + this.channels = channels; + this.limit = limit; + this.ascending = ascending; + } + + @Override + public TopListIntAggregatorFunction aggregator(DriverContext driverContext) { + return TopListIntAggregatorFunction.create(driverContext, channels, limit, ascending); + } + + @Override + public TopListIntGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return TopListIntGroupingAggregatorFunction.create(channels, driverContext, limit, ascending); + } + + @Override + public String describe() { + return "top_list of ints"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..820ebb95e530c --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListIntGroupingAggregatorFunction.java @@ -0,0 +1,200 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link TopListIntAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListIntGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.INT) ); + + private final TopListIntAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int limit; + + private final boolean ascending; + + public TopListIntGroupingAggregatorFunction(List channels, + TopListIntAggregator.GroupingState state, DriverContext driverContext, int limit, + boolean ascending) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListIntGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int limit, boolean ascending) { + return new TopListIntGroupingAggregatorFunction(channels, TopListIntAggregator.initGrouping(driverContext.bigArrays(), limit, ascending), driverContext, limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + IntBlock valuesBlock = page.getBlock(channels.get(0)); + IntVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListIntAggregator.combine(state, groupId, values.getInt(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, IntVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, IntBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListIntAggregator.combine(state, groupId, values.getInt(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, IntVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + TopListIntAggregator.combine(state, groupId, values.getInt(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + IntBlock topList = (IntBlock) topListUncast; + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListIntAggregator.combineIntermediate(state, groupId, topList, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + TopListIntAggregator.GroupingState inState = ((TopListIntGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + TopListIntAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = TopListIntAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunction.java new file mode 100644 index 0000000000000..1a09a1a860e2f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunction.java @@ -0,0 +1,126 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link TopListLongAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListLongAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.LONG) ); + + private final DriverContext driverContext; + + private final TopListLongAggregator.SingleState state; + + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListLongAggregatorFunction(DriverContext driverContext, List channels, + TopListLongAggregator.SingleState state, int limit, boolean ascending) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListLongAggregatorFunction create(DriverContext driverContext, + List channels, int limit, boolean ascending) { + return new TopListLongAggregatorFunction(driverContext, channels, TopListLongAggregator.initSingle(driverContext.bigArrays(), limit, ascending), limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + LongBlock block = page.getBlock(channels.get(0)); + LongVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(LongVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + TopListLongAggregator.combine(state, vector.getLong(i)); + } + } + + private void addRawBlock(LongBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + TopListLongAggregator.combine(state, block.getLong(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + LongBlock topList = (LongBlock) topListUncast; + assert topList.getPositionCount() == 1; + TopListLongAggregator.combineIntermediate(state, topList); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = TopListLongAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..617895fbff1a3 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunctionSupplier.java @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link TopListLongAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListLongAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + private final int limit; + + private final boolean ascending; + + public TopListLongAggregatorFunctionSupplier(List channels, int limit, + boolean ascending) { + this.channels = channels; + this.limit = limit; + this.ascending = ascending; + } + + @Override + public TopListLongAggregatorFunction aggregator(DriverContext driverContext) { + return TopListLongAggregatorFunction.create(driverContext, channels, limit, ascending); + } + + @Override + public TopListLongGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return TopListLongGroupingAggregatorFunction.create(channels, driverContext, limit, ascending); + } + + @Override + public String describe() { + return "top_list of longs"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..cadb48b7d29d4 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/TopListLongGroupingAggregatorFunction.java @@ -0,0 +1,202 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link TopListLongAggregator}. + * This class is generated. Do not edit it. + */ +public final class TopListLongGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("topList", ElementType.LONG) ); + + private final TopListLongAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + private final int limit; + + private final boolean ascending; + + public TopListLongGroupingAggregatorFunction(List channels, + TopListLongAggregator.GroupingState state, DriverContext driverContext, int limit, + boolean ascending) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + this.limit = limit; + this.ascending = ascending; + } + + public static TopListLongGroupingAggregatorFunction create(List channels, + DriverContext driverContext, int limit, boolean ascending) { + return new TopListLongGroupingAggregatorFunction(channels, TopListLongAggregator.initGrouping(driverContext.bigArrays(), limit, ascending), driverContext, limit, ascending); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + LongBlock valuesBlock = page.getBlock(channels.get(0)); + LongVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, LongBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListLongAggregator.combine(state, groupId, values.getLong(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, LongVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, LongBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + TopListLongAggregator.combine(state, groupId, values.getLong(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, LongVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + TopListLongAggregator.combine(state, groupId, values.getLong(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block topListUncast = page.getBlock(channels.get(0)); + if (topListUncast.areAllValuesNull()) { + return; + } + LongBlock topList = (LongBlock) topListUncast; + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + TopListLongAggregator.combineIntermediate(state, groupId, topList, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + TopListLongAggregator.GroupingState inState = ((TopListLongGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + TopListLongAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = TopListLongAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java new file mode 100644 index 0000000000000..c7385e87bfbf2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunction.java @@ -0,0 +1,120 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunction} implementation for {@link ValuesFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class ValuesFloatAggregatorFunction implements AggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("values", ElementType.FLOAT) ); + + private final DriverContext driverContext; + + private final ValuesFloatAggregator.SingleState state; + + private final List channels; + + public ValuesFloatAggregatorFunction(DriverContext driverContext, List channels, + ValuesFloatAggregator.SingleState state) { + this.driverContext = driverContext; + this.channels = channels; + this.state = state; + } + + public static ValuesFloatAggregatorFunction create(DriverContext driverContext, + List channels) { + return new ValuesFloatAggregatorFunction(driverContext, channels, ValuesFloatAggregator.initSingle(driverContext.bigArrays())); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public void addRawInput(Page page) { + FloatBlock block = page.getBlock(channels.get(0)); + FloatVector vector = block.asVector(); + if (vector != null) { + addRawVector(vector); + } else { + addRawBlock(block); + } + } + + private void addRawVector(FloatVector vector) { + for (int i = 0; i < vector.getPositionCount(); i++) { + ValuesFloatAggregator.combine(state, vector.getFloat(i)); + } + } + + private void addRawBlock(FloatBlock block) { + for (int p = 0; p < block.getPositionCount(); p++) { + if (block.isNull(p)) { + continue; + } + int start = block.getFirstValueIndex(p); + int end = start + block.getValueCount(p); + for (int i = start; i < end; i++) { + ValuesFloatAggregator.combine(state, block.getFloat(i)); + } + } + } + + @Override + public void addIntermediateInput(Page page) { + assert channels.size() == intermediateBlockCount(); + assert page.getBlockCount() >= channels.get(0) + intermediateStateDesc().size(); + Block valuesUncast = page.getBlock(channels.get(0)); + if (valuesUncast.areAllValuesNull()) { + return; + } + FloatBlock values = (FloatBlock) valuesUncast; + assert values.getPositionCount() == 1; + ValuesFloatAggregator.combineIntermediate(state, values); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + state.toIntermediate(blocks, offset, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = ValuesFloatAggregator.evaluateFinal(state, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionSupplier.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionSupplier.java new file mode 100644 index 0000000000000..b4b0c2f1a0444 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionSupplier.java @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.util.List; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link AggregatorFunctionSupplier} implementation for {@link ValuesFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class ValuesFloatAggregatorFunctionSupplier implements AggregatorFunctionSupplier { + private final List channels; + + public ValuesFloatAggregatorFunctionSupplier(List channels) { + this.channels = channels; + } + + @Override + public ValuesFloatAggregatorFunction aggregator(DriverContext driverContext) { + return ValuesFloatAggregatorFunction.create(driverContext, channels); + } + + @Override + public ValuesFloatGroupingAggregatorFunction groupingAggregator(DriverContext driverContext) { + return ValuesFloatGroupingAggregatorFunction.create(channels, driverContext); + } + + @Override + public String describe() { + return "values of floats"; + } +} diff --git a/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java new file mode 100644 index 0000000000000..54cc06072cd24 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/generated/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunction.java @@ -0,0 +1,195 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.compute.aggregation; + +import java.lang.Integer; +import java.lang.Override; +import java.lang.String; +import java.lang.StringBuilder; +import java.util.List; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.DriverContext; + +/** + * {@link GroupingAggregatorFunction} implementation for {@link ValuesFloatAggregator}. + * This class is generated. Do not edit it. + */ +public final class ValuesFloatGroupingAggregatorFunction implements GroupingAggregatorFunction { + private static final List INTERMEDIATE_STATE_DESC = List.of( + new IntermediateStateDesc("values", ElementType.FLOAT) ); + + private final ValuesFloatAggregator.GroupingState state; + + private final List channels; + + private final DriverContext driverContext; + + public ValuesFloatGroupingAggregatorFunction(List channels, + ValuesFloatAggregator.GroupingState state, DriverContext driverContext) { + this.channels = channels; + this.state = state; + this.driverContext = driverContext; + } + + public static ValuesFloatGroupingAggregatorFunction create(List channels, + DriverContext driverContext) { + return new ValuesFloatGroupingAggregatorFunction(channels, ValuesFloatAggregator.initGrouping(driverContext.bigArrays()), driverContext); + } + + public static List intermediateStateDesc() { + return INTERMEDIATE_STATE_DESC; + } + + @Override + public int intermediateBlockCount() { + return INTERMEDIATE_STATE_DESC.size(); + } + + @Override + public GroupingAggregatorFunction.AddInput prepareProcessPage(SeenGroupIds seenGroupIds, + Page page) { + FloatBlock valuesBlock = page.getBlock(channels.get(0)); + FloatVector valuesVector = valuesBlock.asVector(); + if (valuesVector == null) { + if (valuesBlock.mayHaveNulls()) { + state.enableGroupIdTracking(seenGroupIds); + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesBlock); + } + }; + } + return new GroupingAggregatorFunction.AddInput() { + @Override + public void add(int positionOffset, IntBlock groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + + @Override + public void add(int positionOffset, IntVector groupIds) { + addRawInput(positionOffset, groupIds, valuesVector); + } + }; + } + + private void addRawInput(int positionOffset, IntVector groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + ValuesFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + + private void addRawInput(int positionOffset, IntVector groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + ValuesFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatBlock values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + if (values.isNull(groupPosition + positionOffset)) { + continue; + } + int valuesStart = values.getFirstValueIndex(groupPosition + positionOffset); + int valuesEnd = valuesStart + values.getValueCount(groupPosition + positionOffset); + for (int v = valuesStart; v < valuesEnd; v++) { + ValuesFloatAggregator.combine(state, groupId, values.getFloat(v)); + } + } + } + } + + private void addRawInput(int positionOffset, IntBlock groups, FloatVector values) { + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + if (groups.isNull(groupPosition)) { + continue; + } + int groupStart = groups.getFirstValueIndex(groupPosition); + int groupEnd = groupStart + groups.getValueCount(groupPosition); + for (int g = groupStart; g < groupEnd; g++) { + int groupId = Math.toIntExact(groups.getInt(g)); + ValuesFloatAggregator.combine(state, groupId, values.getFloat(groupPosition + positionOffset)); + } + } + } + + @Override + public void addIntermediateInput(int positionOffset, IntVector groups, Page page) { + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + assert channels.size() == intermediateBlockCount(); + Block valuesUncast = page.getBlock(channels.get(0)); + if (valuesUncast.areAllValuesNull()) { + return; + } + FloatBlock values = (FloatBlock) valuesUncast; + for (int groupPosition = 0; groupPosition < groups.getPositionCount(); groupPosition++) { + int groupId = Math.toIntExact(groups.getInt(groupPosition)); + ValuesFloatAggregator.combineIntermediate(state, groupId, values, groupPosition + positionOffset); + } + } + + @Override + public void addIntermediateRowInput(int groupId, GroupingAggregatorFunction input, int position) { + if (input.getClass() != getClass()) { + throw new IllegalArgumentException("expected " + getClass() + "; got " + input.getClass()); + } + ValuesFloatAggregator.GroupingState inState = ((ValuesFloatGroupingAggregatorFunction) input).state; + state.enableGroupIdTracking(new SeenGroupIds.Empty()); + ValuesFloatAggregator.combineStates(state, groupId, inState, position); + } + + @Override + public void evaluateIntermediate(Block[] blocks, int offset, IntVector selected) { + state.toIntermediate(blocks, offset, selected, driverContext); + } + + @Override + public void evaluateFinal(Block[] blocks, int offset, IntVector selected, + DriverContext driverContext) { + blocks[offset] = ValuesFloatAggregator.evaluateFinal(state, selected, driverContext); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getClass().getSimpleName()).append("["); + sb.append("channels=").append(channels); + sb.append("]"); + return sb.toString(); + } + + @Override + public void close() { + state.close(); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/module-info.java b/x-pack/plugin/esql/compute/src/main/java/module-info.java index 3772d6c83f5aa..dc8cda0fbe3c8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/module-info.java +++ b/x-pack/plugin/esql/compute/src/main/java/module-info.java @@ -30,4 +30,5 @@ exports org.elasticsearch.compute.operator.topn; exports org.elasticsearch.compute.operator.mvdedupe; exports org.elasticsearch.compute.aggregation.table; + exports org.elasticsearch.compute.data.sort; } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregator.java new file mode 100644 index 0000000000000..2159f0864e1cf --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregator.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.DriverContext; + +@Aggregator({ @IntermediateState(name = "hll", type = "BYTES_REF") }) +@GroupingAggregator +public class CountDistinctFloatAggregator { + + public static HllStates.SingleState initSingle(BigArrays bigArrays, int precision) { + return new HllStates.SingleState(bigArrays, precision); + } + + public static void combine(HllStates.SingleState current, float v) { + current.collect(v); + } + + public static void combineIntermediate(HllStates.SingleState current, BytesRef inValue) { + current.merge(0, inValue, 0); + } + + public static Block evaluateFinal(HllStates.SingleState state, DriverContext driverContext) { + long result = state.cardinality(); + return driverContext.blockFactory().newConstantLongBlockWith(result, 1); + } + + public static HllStates.GroupingState initGrouping(BigArrays bigArrays, int precision) { + return new HllStates.GroupingState(bigArrays, precision); + } + + public static void combine(HllStates.GroupingState current, int groupId, float v) { + current.collect(groupId, v); + } + + public static void combineIntermediate(HllStates.GroupingState current, int groupId, BytesRef inValue) { + current.merge(groupId, inValue, 0); + } + + public static void combineStates( + HllStates.GroupingState current, + int currentGroupId, + HllStates.GroupingState state, + int statePosition + ) { + current.merge(currentGroupId, state.hll, statePosition); + } + + public static Block evaluateFinal(HllStates.GroupingState state, IntVector selected, DriverContext driverContext) { + try (LongBlock.Builder builder = driverContext.blockFactory().newLongBlockBuilder(selected.getPositionCount())) { + for (int i = 0; i < selected.getPositionCount(); i++) { + int group = selected.getInt(i); + long count = state.cardinality(group); + builder.appendLong(count); + } + return builder.build(); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxFloatAggregator.java new file mode 100644 index 0000000000000..eea436541069e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MaxFloatAggregator.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +@Aggregator({ @IntermediateState(name = "max", type = "FLOAT"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class MaxFloatAggregator { + + public static float init() { + return Float.MIN_VALUE; + } + + public static float combine(float current, float v) { + return Math.max(current, v); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregator.java new file mode 100644 index 0000000000000..b81cc945f0695 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregator.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; + +@Aggregator({ @IntermediateState(name = "quart", type = "BYTES_REF") }) +@GroupingAggregator +class MedianAbsoluteDeviationFloatAggregator { + + public static QuantileStates.SingleState initSingle() { + return new QuantileStates.SingleState(QuantileStates.MEDIAN); + } + + public static void combine(QuantileStates.SingleState current, float v) { + current.add(v); + } + + public static void combineIntermediate(QuantileStates.SingleState state, BytesRef inValue) { + state.add(inValue); + } + + public static Block evaluateFinal(QuantileStates.SingleState state, DriverContext driverContext) { + return state.evaluateMedianAbsoluteDeviation(driverContext); + } + + public static QuantileStates.GroupingState initGrouping(BigArrays bigArrays) { + return new QuantileStates.GroupingState(bigArrays, QuantileStates.MEDIAN); + } + + public static void combine(QuantileStates.GroupingState state, int groupId, float v) { + state.add(groupId, v); + } + + public static void combineIntermediate(QuantileStates.GroupingState state, int groupId, BytesRef inValue) { + state.add(groupId, inValue); + } + + public static void combineStates( + QuantileStates.GroupingState current, + int currentGroupId, + QuantileStates.GroupingState state, + int statePosition + ) { + current.add(currentGroupId, state.getOrNull(statePosition)); + } + + public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected, DriverContext driverContext) { + return state.evaluateMedianAbsoluteDeviation(selected, driverContext); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinFloatAggregator.java new file mode 100644 index 0000000000000..9ea52eab846c1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/MinFloatAggregator.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +@Aggregator({ @IntermediateState(name = "min", type = "FLOAT"), @IntermediateState(name = "seen", type = "BOOLEAN") }) +@GroupingAggregator +class MinFloatAggregator { + + public static float init() { + return Float.POSITIVE_INFINITY; + } + + public static float combine(float current, float v) { + return Math.min(current, v); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregator.java new file mode 100644 index 0000000000000..37b68b3c31335 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregator.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.operator.DriverContext; + +@Aggregator({ @IntermediateState(name = "quart", type = "BYTES_REF") }) +@GroupingAggregator +class PercentileFloatAggregator { + + public static QuantileStates.SingleState initSingle(double percentile) { + return new QuantileStates.SingleState(percentile); + } + + public static void combine(QuantileStates.SingleState current, float v) { + current.add(v); + } + + public static void combineIntermediate(QuantileStates.SingleState state, BytesRef inValue) { + state.add(inValue); + } + + public static Block evaluateFinal(QuantileStates.SingleState state, DriverContext driverContext) { + return state.evaluatePercentile(driverContext); + } + + public static QuantileStates.GroupingState initGrouping(BigArrays bigArrays, double percentile) { + return new QuantileStates.GroupingState(bigArrays, percentile); + } + + public static void combine(QuantileStates.GroupingState state, int groupId, float v) { + state.add(groupId, v); + } + + public static void combineIntermediate(QuantileStates.GroupingState state, int groupId, BytesRef inValue) { + state.add(groupId, inValue); + } + + public static void combineStates( + QuantileStates.GroupingState current, + int currentGroupId, + QuantileStates.GroupingState state, + int statePosition + ) { + current.add(currentGroupId, state.getOrNull(statePosition)); + } + + public static Block evaluateFinal(QuantileStates.GroupingState state, IntVector selected, DriverContext driverContext) { + return state.evaluatePercentile(selected, driverContext); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/SumFloatAggregator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/SumFloatAggregator.java new file mode 100644 index 0000000000000..ea6b55b949f15 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/SumFloatAggregator.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; + +@Aggregator( + { + @IntermediateState(name = "value", type = "DOUBLE"), + @IntermediateState(name = "delta", type = "DOUBLE"), + @IntermediateState(name = "seen", type = "BOOLEAN") } +) +@GroupingAggregator +class SumFloatAggregator extends SumDoubleAggregator { + + public static void combine(SumState current, float v) { + current.add(v); + } + + public static void combine(GroupingSumState current, int groupId, float v) { + current.add(v, groupId); + } + +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st index 246aebe2c08ec..18686928f14a8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ArrayState.java.st @@ -17,7 +17,7 @@ import org.elasticsearch.compute.data.$Type$Block; $if(int)$ import org.elasticsearch.compute.data.$Type$Vector; $endif$ -$if(double)$ +$if(double||float)$ import org.elasticsearch.compute.data.IntVector; $endif$ import org.elasticsearch.compute.operator.DriverContext; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st index 212a017cb300d..2581d3ebbf80b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-RateAggregator.java.st @@ -18,7 +18,9 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.DoubleVector; -$if(int)$ +$if(float)$ +import org.elasticsearch.compute.data.FloatBlock; +$elseif(int)$ import org.elasticsearch.compute.data.IntBlock; $endif$ import org.elasticsearch.compute.data.IntVector; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopListAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopListAggregator.java.st new file mode 100644 index 0000000000000..810311154503e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopListAggregator.java.st @@ -0,0 +1,142 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.compute.ann.Aggregator; +import org.elasticsearch.compute.ann.GroupingAggregator; +import org.elasticsearch.compute.ann.IntermediateState; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +$if(!long)$ +import org.elasticsearch.compute.data.$Type$Block; +$endif$ +import org.elasticsearch.compute.data.IntVector; +$if(long)$ +import org.elasticsearch.compute.data.$Type$Block; +$endif$ +import org.elasticsearch.compute.data.sort.$Type$BucketedSort; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.search.sort.SortOrder; + +/** + * Aggregates the top N field values for $type$. + */ +@Aggregator({ @IntermediateState(name = "topList", type = "$TYPE$_BLOCK") }) +@GroupingAggregator +class TopList$Type$Aggregator { + public static SingleState initSingle(BigArrays bigArrays, int limit, boolean ascending) { + return new SingleState(bigArrays, limit, ascending); + } + + public static void combine(SingleState state, $type$ v) { + state.add(v); + } + + public static void combineIntermediate(SingleState state, $Type$Block values) { + int start = values.getFirstValueIndex(0); + int end = start + values.getValueCount(0); + for (int i = start; i < end; i++) { + combine(state, values.get$Type$(i)); + } + } + + public static Block evaluateFinal(SingleState state, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory()); + } + + public static GroupingState initGrouping(BigArrays bigArrays, int limit, boolean ascending) { + return new GroupingState(bigArrays, limit, ascending); + } + + public static void combine(GroupingState state, int groupId, $type$ v) { + state.add(groupId, v); + } + + public static void combineIntermediate(GroupingState state, int groupId, $Type$Block values, int valuesPosition) { + int start = values.getFirstValueIndex(valuesPosition); + int end = start + values.getValueCount(valuesPosition); + for (int i = start; i < end; i++) { + combine(state, groupId, values.get$Type$(i)); + } + } + + public static void combineStates(GroupingState current, int groupId, GroupingState state, int statePosition) { + current.merge(groupId, state, statePosition); + } + + public static Block evaluateFinal(GroupingState state, IntVector selected, DriverContext driverContext) { + return state.toBlock(driverContext.blockFactory(), selected); + } + + public static class GroupingState implements Releasable { + private final $Type$BucketedSort sort; + + private GroupingState(BigArrays bigArrays, int limit, boolean ascending) { + this.sort = new $Type$BucketedSort(bigArrays, ascending ? SortOrder.ASC : SortOrder.DESC, limit); + } + + public void add(int groupId, $type$ value) { + sort.collect(value, groupId); + } + + public void merge(int groupId, GroupingState other, int otherGroupId) { + sort.merge(groupId, other.sort, otherGroupId); + } + + void toIntermediate(Block[] blocks, int offset, IntVector selected, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory(), selected); + } + + Block toBlock(BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + void enableGroupIdTracking(SeenGroupIds seen) { + // we figure out seen values from nulls on the values block + } + + @Override + public void close() { + Releasables.closeExpectNoException(sort); + } + } + + public static class SingleState implements Releasable { + private final GroupingState internalState; + + private SingleState(BigArrays bigArrays, int limit, boolean ascending) { + this.internalState = new GroupingState(bigArrays, limit, ascending); + } + + public void add($type$ value) { + internalState.add(0, value); + } + + public void merge(GroupingState other) { + internalState.merge(0, other, 0); + } + + void toIntermediate(Block[] blocks, int offset, DriverContext driverContext) { + blocks[offset] = toBlock(driverContext.blockFactory()); + } + + Block toBlock(BlockFactory blockFactory) { + try (var intValues = blockFactory.newConstantIntVector(0, 1)) { + return internalState.toBlock(blockFactory, intValues); + } + } + + @Override + public void close() { + Releasables.closeExpectNoException(internalState); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st index f9b15ccd34092..ea62dcf295825 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-ValuesAggregator.java.st @@ -27,7 +27,7 @@ import org.elasticsearch.compute.ann.GroupingAggregator; import org.elasticsearch.compute.ann.IntermediateState; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; -$if(int||double||BytesRef)$ +$if(int||double||float||BytesRef)$ import org.elasticsearch.compute.data.$Type$Block; $endif$ import org.elasticsearch.compute.data.IntVector; @@ -55,7 +55,9 @@ class Values$Type$Aggregator { } public static void combine(SingleState state, $type$ v) { -$if(double)$ +$if(float)$ + state.values.add(Float.floatToIntBits(v)); +$elseif(double)$ state.values.add(Double.doubleToLongBits(v)); $else$ state.values.add(v); @@ -98,6 +100,12 @@ $elseif(int)$ * the top 32 bits for the group, the bottom 32 for the value. */ state.values.add((((long) groupId) << Integer.SIZE) | (v & 0xFFFFFFFFL)); +$elseif(float)$ + /* + * Encode the groupId and value into a single long - + * the top 32 bits for the group, the bottom 32 for the value. + */ + state.values.add((((long) groupId) << Float.SIZE) | (Float.floatToIntBits(v) & 0xFFFFFFFFL)); $endif$ } @@ -132,6 +140,11 @@ $elseif(int)$ int group = (int) (both >>> Integer.SIZE); if (group == statePosition) { int value = (int) both; +$elseif(float)$ + long both = state.values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group == statePosition) { + float value = Float.intBitsToFloat((int) both); $endif$ combine(current, currentGroupId, $if(BytesRef)$state.bytes.get(value, scratch)$else$value$endif$); } @@ -172,6 +185,8 @@ $endif$ if (values.size() == 1) { $if(long)$ return blockFactory.newConstantLongBlockWith(values.get(0), 1); +$elseif(float)$ + return blockFactory.newConstantFloatBlockWith(Float.intBitsToFloat((int) values.get(0)), 1); $elseif(double)$ return blockFactory.newConstantDoubleBlockWith(Double.longBitsToDouble(values.get(0)), 1); $elseif(int)$ @@ -185,6 +200,8 @@ $endif$ for (int id = 0; id < values.size(); id++) { $if(long)$ builder.appendLong(values.get(id)); +$elseif(float)$ + builder.appendFloat(Float.intBitsToFloat((int) values.get(id))); $elseif(double)$ builder.appendDouble(Double.longBitsToDouble(values.get(id))); $elseif(int)$ @@ -219,7 +236,7 @@ $elseif(BytesRef)$ private final LongLongHash values; private final BytesRefHash bytes; -$elseif(int)$ +$elseif(int||float)$ private final LongHash values; $endif$ @@ -229,7 +246,7 @@ $if(long||double)$ $elseif(BytesRef)$ values = new LongLongHash(1, bigArrays); bytes = new BytesRefHash(1, bigArrays); -$elseif(int)$ +$elseif(int||float)$ values = new LongHash(1, bigArrays); $endif$ } @@ -262,6 +279,11 @@ $if(long||BytesRef)$ $elseif(double)$ if (values.getKey1(id) == selectedGroup) { double value = Double.longBitsToDouble(values.getKey2(id)); +$elseif(float)$ + long both = values.get(id); + int group = (int) (both >>> Float.SIZE); + if (group == selectedGroup) { + float value = Float.intBitsToFloat((int) both); $elseif(int)$ long both = values.get(id); int group = (int) (both >>> Integer.SIZE); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java index ca3ce1349c47f..282bc9064b308 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Block.java @@ -276,6 +276,7 @@ static List getNamedWriteables() { return List.of( IntBlock.ENTRY, LongBlock.ENTRY, + FloatBlock.ENTRY, DoubleBlock.ENTRY, BytesRefBlock.ENTRY, BooleanBlock.ENTRY, diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java index 7b91ff6a645ae..155898ebdc6c8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockFactory.java @@ -235,6 +235,58 @@ public IntVector newConstantIntVector(int value, int positions) { return v; } + public FloatBlock.Builder newFloatBlockBuilder(int estimatedSize) { + return new FloatBlockBuilder(estimatedSize, this); + } + + public final FloatBlock newFloatArrayBlock(float[] values, int pc, int[] firstValueIndexes, BitSet nulls, MvOrdering mvOrdering) { + return newFloatArrayBlock(values, pc, firstValueIndexes, nulls, mvOrdering, 0L); + } + + public FloatBlock newFloatArrayBlock(float[] values, int pc, int[] fvi, BitSet nulls, MvOrdering mvOrdering, long preAdjustedBytes) { + var b = new FloatArrayBlock(values, pc, fvi, nulls, mvOrdering, this); + adjustBreaker(b.ramBytesUsed() - preAdjustedBytes); + return b; + } + + public FloatVector.Builder newFloatVectorBuilder(int estimatedSize) { + return new FloatVectorBuilder(estimatedSize, this); + } + + /** + * Build a {@link FloatVector.FixedBuilder} that never grows. + */ + public FloatVector.FixedBuilder newFloatVectorFixedBuilder(int size) { + return new FloatVectorFixedBuilder(size, this); + } + + public final FloatVector newFloatArrayVector(float[] values, int positionCount) { + return newFloatArrayVector(values, positionCount, 0L); + } + + public FloatVector newFloatArrayVector(float[] values, int positionCount, long preAdjustedBytes) { + var b = new FloatArrayVector(values, positionCount, this); + adjustBreaker(b.ramBytesUsed() - preAdjustedBytes); + return b; + } + + public final FloatBlock newConstantFloatBlockWith(float value, int positions) { + return newConstantFloatBlockWith(value, positions, 0L); + } + + public FloatBlock newConstantFloatBlockWith(float value, int positions, long preAdjustedBytes) { + var b = new ConstantFloatVector(value, positions, this).asBlock(); + adjustBreaker(b.ramBytesUsed() - preAdjustedBytes); + return b; + } + + public FloatVector newConstantFloatVector(float value, int positions) { + adjustBreaker(ConstantFloatVector.RAM_BYTES_USED); + var v = new ConstantFloatVector(value, positions, this); + assert v.ramBytesUsed() == ConstantFloatVector.RAM_BYTES_USED; + return v; + } + public LongBlock.Builder newLongBlockBuilder(int estimatedSize) { return new LongBlockBuilder(estimatedSize, this); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index 7e846bd32e3cb..a697a3f6c15fa 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -208,6 +208,7 @@ public static void appendValue(Block.Builder builder, Object val, ElementType ty case LONG -> ((LongBlock.Builder) builder).appendLong((Long) val); case INT -> ((IntBlock.Builder) builder).appendInt((Integer) val); case BYTES_REF -> ((BytesRefBlock.Builder) builder).appendBytesRef(toBytesRef(val)); + case FLOAT -> ((FloatBlock.Builder) builder).appendFloat((Float) val); case DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble((Double) val); case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean((Boolean) val); default -> throw new UnsupportedOperationException("unsupported element type [" + type + "]"); @@ -265,6 +266,7 @@ private static Object valueAtOffset(Block block, int offset) { case BOOLEAN -> ((BooleanBlock) block).getBoolean(offset); case BYTES_REF -> BytesRef.deepCopyOf(((BytesRefBlock) block).getBytesRef(offset, new BytesRef())); case DOUBLE -> ((DoubleBlock) block).getDouble(offset); + case FLOAT -> ((FloatBlock) block).getFloat(offset); case INT -> ((IntBlock) block).getInt(offset); case LONG -> ((LongBlock) block).getLong(offset); case NULL -> null; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java index ae14033a00b3e..2c0f4c8946753 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullBlock.java @@ -25,6 +25,7 @@ final class ConstantNullBlock extends AbstractNonThreadSafeRefCounted BooleanBlock, IntBlock, LongBlock, + FloatBlock, DoubleBlock, BytesRefBlock { @@ -221,6 +222,12 @@ public BytesRef getBytesRef(int valueIndex, BytesRef dest) { throw new UnsupportedOperationException("null block"); } + @Override + public float getFloat(int valueIndex) { + assert false : "null block"; + throw new UnsupportedOperationException("null block"); + } + @Override public double getDouble(int valueIndex) { assert false : "null block"; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java index a8a6dbaf382f9..b053267ba0e0f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ConstantNullVector.java @@ -17,7 +17,14 @@ /** * This vector is never instantiated. This class serves as a type holder for {@link ConstantNullBlock#asVector()}. */ -public final class ConstantNullVector extends AbstractVector implements BooleanVector, IntVector, LongVector, DoubleVector, BytesRefVector { +public final class ConstantNullVector extends AbstractVector + implements + BooleanVector, + BytesRefVector, + DoubleVector, + FloatVector, + IntVector, + LongVector { private ConstantNullVector(int positionCount, BlockFactory blockFactory) { super(positionCount, blockFactory); @@ -65,6 +72,12 @@ public BytesRef getBytesRef(int position, BytesRef dest) { throw new UnsupportedOperationException("null vector"); } + @Override + public float getFloat(int position) { + assert false : "null vector"; + throw new UnsupportedOperationException("null vector"); + } + @Override public double getDouble(int position) { assert false : "null vector"; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java index 5796153748817..52c84c80610d2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/ElementType.java @@ -16,6 +16,7 @@ public enum ElementType { BOOLEAN(BlockFactory::newBooleanBlockBuilder), INT(BlockFactory::newIntBlockBuilder), LONG(BlockFactory::newLongBlockBuilder), + FLOAT(BlockFactory::newFloatBlockBuilder), DOUBLE(BlockFactory::newDoubleBlockBuilder), /** * Blocks containing only null values. @@ -62,6 +63,8 @@ public static ElementType fromJava(Class type) { elementType = INT; } else if (type == Long.class) { elementType = LONG; + } else if (type == Float.class) { + elementType = FLOAT; } else if (type == Double.class) { elementType = DOUBLE; } else if (type == String.class || type == BytesRef.class) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java index 4d41ab27312c3..2e46735bd5bd1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/Page.java @@ -83,7 +83,9 @@ private Page(boolean copyBlocks, int positionCount, Block[] blocks) { private Page(Page prev, Block[] toAdd) { for (Block block : toAdd) { if (prev.positionCount != block.getPositionCount()) { - throw new IllegalArgumentException("Block [" + block + "] does not have same position count"); + throw new IllegalArgumentException( + "Block [" + block + "] does not have same position count: " + block.getPositionCount() + " != " + prev.positionCount + ); } } this.positionCount = prev.positionCount; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st index 30ef9e799cf11..85bf2b086e3b2 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-BigArrayVector.java.st @@ -46,6 +46,9 @@ $endif$ } static $Type$BigArrayVector readArrayVector(int positions, StreamInput in, BlockFactory blockFactory) throws IOException { +$if(float)$ + throw new UnsupportedOperationException(); +$else$ $if(boolean)$ $Array$ values = new BitArray(blockFactory.bigArrays(), true, in); $else$ @@ -65,10 +68,15 @@ $endif$ values.close(); } } +$endif$ } void writeArrayVector(int positions, StreamOutput out) throws IOException { +$if(float)$ + throw new UnsupportedOperationException(); +$else$ values.writeTo(out); +$endif$ } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st index 6d62d44f99e66..dc6f4ee1003cf 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Block.java.st @@ -246,6 +246,8 @@ $elseif(boolean)$ result = 31 * result + Boolean.hashCode(block.getBoolean(firstValueIdx + valueIndex)); $elseif(int)$ result = 31 * result + block.getInt(firstValueIdx + valueIndex); +$elseif(float)$ + result = 31 * result + Float.floatToIntBits(block.getFloat(pos)); $elseif(long)$ long element = block.getLong(firstValueIdx + valueIndex); result = 31 * result + (int) (element ^ (element >>> 32)); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st index 0113f4940adb5..28332648b5d3f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/X-Vector.java.st @@ -121,6 +121,8 @@ $elseif(boolean)$ result = 31 * result + Boolean.hashCode(vector.getBoolean(pos)); $elseif(int)$ result = 31 * result + vector.getInt(pos); +$elseif(float)$ + result = 31 * result + Float.floatToIntBits(vector.getFloat(pos)); $elseif(long)$ long element = vector.getLong(pos); result = 31 * result + (int) (element ^ (element >>> 32)); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st new file mode 100644 index 0000000000000..6587743e34b6f --- /dev/null +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/X-BucketedSort.java.st @@ -0,0 +1,350 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.$Type$Array; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.util.Arrays; +import java.util.stream.IntStream; + +/** + * Aggregates the top N $type$ values per bucket. + * See {@link BucketedSort} for more information. + * This class is generated. Edit @{code X-BucketedSort.java.st} instead of this file. + */ +public class $Type$BucketedSort implements Releasable { + + private final BigArrays bigArrays; + private final SortOrder order; + private final int bucketSize; + /** + * {@code true} if the bucket is in heap mode, {@code false} if + * it is still gathering. + */ + private final BitArray heapMode; + /** + * An array containing all the values on all buckets. The structure is as follows: + *

    + * For each bucket, there are bucketSize elements, based on the bucket id (0, 1, 2...). + * Then, for each bucket, it can be in 2 states: + *

    + *
      + *
    • + * Gather mode: All buckets start in gather mode, and remain here while they have less than bucketSize elements. + * In gather mode, the elements are stored in the array from the highest index to the lowest index. + * The lowest index contains the offset to the next slot to be filled. + *

      + * This allows us to insert elements in O(1) time. + *

      + *

      + * When the bucketSize-th element is collected, the bucket transitions to heap mode, by heapifying its contents. + *

      + *
      • + *
    • + * Heap mode: The bucket slots are organized as a min heap structure. + *

      + * The root of the heap is the minimum value in the bucket, + * which allows us to quickly discard new values that are not in the top N. + *

      + *
      • + *
    + */ + private $Type$Array values; + + public $Type$BucketedSort(BigArrays bigArrays, SortOrder order, int bucketSize) { + this.bigArrays = bigArrays; + this.order = order; + this.bucketSize = bucketSize; + heapMode = new BitArray(0, bigArrays); + + boolean success = false; + try { + values = bigArrays.new$Type$Array(0, false); + success = true; + } finally { + if (success == false) { + close(); + } + } + } + + /** + * Collects a {@code value} into a {@code bucket}. + *

    + * It may or may not be inserted in the heap, depending on if it is better than the current root. + *

    + */ + public void collect($type$ value, int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (inHeapMode(bucket)) { + if (betterThan(value, values.get(rootIndex))) { + values.set(rootIndex, value); + downHeap(rootIndex, 0); + } + return; + } + // Gathering mode + long requiredSize = rootIndex + bucketSize; + if (values.size() < requiredSize) { + grow(requiredSize); + } + int next = getNextGatherOffset(rootIndex); + assert 0 <= next && next < bucketSize + : "Expected next to be in the range of valid buckets [0 <= " + next + " < " + bucketSize + "]"; + long index = next + rootIndex; + values.set(index, value); + if (next == 0) { + heapMode.set(bucket); + heapify(rootIndex); + } else { + setNextGatherOffset(rootIndex, next - 1); + } + } + + /** + * The order of the sort. + */ + public SortOrder getOrder() { + return order; + } + + /** + * The number of values to store per bucket. + */ + public int getBucketSize() { + return bucketSize; + } + + /** + * Get the first and last indexes (inclusive, exclusive) of the values for a bucket. + * Returns [0, 0] if the bucket has never been collected. + */ + private Tuple getBucketValuesIndexes(int bucket) { + long rootIndex = (long) bucket * bucketSize; + if (rootIndex >= values.size()) { + // We've never seen this bucket. + return Tuple.tuple(0L, 0L); + } + long start = inHeapMode(bucket) ? rootIndex : (rootIndex + getNextGatherOffset(rootIndex) + 1); + long end = rootIndex + bucketSize; + return Tuple.tuple(start, end); + } + + /** + * Merge the values from {@code other}'s {@code otherGroupId} into {@code groupId}. + */ + public void merge(int groupId, $Type$BucketedSort other, int otherGroupId) { + var otherBounds = other.getBucketValuesIndexes(otherGroupId); + + // TODO: This can be improved for heapified buckets by making use of the heap structures + for (long i = otherBounds.v1(); i < otherBounds.v2(); i++) { + collect(other.values.get(i), groupId); + } + } + + /** + * Creates a block with the values from the {@code selected} groups. + */ + public Block toBlock(BlockFactory blockFactory, IntVector selected) { + // Check if the selected groups are all empty, to avoid allocating extra memory + if (IntStream.range(0, selected.getPositionCount()).map(selected::getInt).noneMatch(bucket -> { + var bounds = this.getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + return size > 0; + })) { + return blockFactory.newConstantNullBlock(selected.getPositionCount()); + } + + // Used to sort the values in the bucket. + var bucketValues = new $type$[bucketSize]; + + try (var builder = blockFactory.new$Type$BlockBuilder(selected.getPositionCount())) { + for (int s = 0; s < selected.getPositionCount(); s++) { + int bucket = selected.getInt(s); + + var bounds = getBucketValuesIndexes(bucket); + var size = bounds.v2() - bounds.v1(); + + if (size == 0) { + builder.appendNull(); + continue; + } + + if (size == 1) { + builder.append$Type$(values.get(bounds.v1())); + continue; + } + + for (int i = 0; i < size; i++) { + bucketValues[i] = values.get(bounds.v1() + i); + } + + // TODO: Make use of heap structures to faster iterate in order instead of copying and sorting + Arrays.sort(bucketValues, 0, (int) size); + + builder.beginPositionEntry(); + if (order == SortOrder.ASC) { + for (int i = 0; i < size; i++) { + builder.append$Type$(bucketValues[i]); + } + } else { + for (int i = (int) size - 1; i >= 0; i--) { + builder.append$Type$(bucketValues[i]); + } + } + builder.endPositionEntry(); + } + return builder.build(); + } + } + + /** + * Is this bucket a min heap {@code true} or in gathering mode {@code false}? + */ + private boolean inHeapMode(int bucket) { + return heapMode.get(bucket); + } + + /** + * Get the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private int getNextGatherOffset(long rootIndex) { +$if(int)$ + return values.get(rootIndex); +$else$ + return (int) values.get(rootIndex); +$endif$ + } + + /** + * Set the next index that should be "gathered" for a bucket rooted + * at {@code rootIndex}. + */ + private void setNextGatherOffset(long rootIndex, int offset) { + values.set(rootIndex, offset); + } + + /** + * {@code true} if the entry at index {@code lhs} is "better" than + * the entry at {@code rhs}. "Better" in this means "lower" for + * {@link SortOrder#ASC} and "higher" for {@link SortOrder#DESC}. + */ + private boolean betterThan($type$ lhs, $type$ rhs) { + return getOrder().reverseMul() * $Wrapper$.compare(lhs, rhs) < 0; + } + + /** + * Swap the data at two indices. + */ + private void swap(long lhs, long rhs) { + var tmp = values.get(lhs); + values.set(lhs, values.get(rhs)); + values.set(rhs, tmp); + } + + /** + * Allocate storage for more buckets and store the "next gather offset" + * for those new buckets. + */ + private void grow(long minSize) { + long oldMax = values.size(); + values = bigArrays.grow(values, minSize); + // Set the next gather offsets for all newly allocated buckets. + setNextGatherOffsets(oldMax - (oldMax % getBucketSize())); + } + + /** + * Maintain the "next gather offsets" for newly allocated buckets. + */ + private void setNextGatherOffsets(long startingAt) { + int nextOffset = getBucketSize() - 1; + for (long bucketRoot = startingAt; bucketRoot < values.size(); bucketRoot += getBucketSize()) { + setNextGatherOffset(bucketRoot, nextOffset); + } + } + + /** + * Heapify a bucket whose entries are in random order. + *

    + * This works by validating the heap property on each node, iterating + * "upwards", pushing any out of order parents "down". Check out the + * wikipedia + * entry on binary heaps for more about this. + *

    + *

    + * While this *looks* like it could easily be {@code O(n * log n)}, it is + * a fairly well studied algorithm attributed to Floyd. There's + * been a bunch of work that puts this at {@code O(n)}, close to 1.88n worst + * case. + *

    + * + * @param rootIndex the index the start of the bucket + */ + private void heapify(long rootIndex) { + int maxParent = bucketSize / 2 - 1; + for (int parent = maxParent; parent >= 0; parent--) { + downHeap(rootIndex, parent); + } + } + + /** + * Correct the heap invariant of a parent and its children. This + * runs in {@code O(log n)} time. + * @param rootIndex index of the start of the bucket + * @param parent Index within the bucket of the parent to check. + * For example, 0 is the "root". + */ + private void downHeap(long rootIndex, int parent) { + while (true) { + long parentIndex = rootIndex + parent; + int worst = parent; + long worstIndex = parentIndex; + int leftChild = parent * 2 + 1; + long leftIndex = rootIndex + leftChild; + if (leftChild < bucketSize) { + if (betterThan(values.get(worstIndex), values.get(leftIndex))) { + worst = leftChild; + worstIndex = leftIndex; + } + int rightChild = leftChild + 1; + long rightIndex = rootIndex + rightChild; + if (rightChild < bucketSize && betterThan(values.get(worstIndex), values.get(rightIndex))) { + worst = rightChild; + worstIndex = rightIndex; + } + } + if (worst == parent) { + break; + } + swap(worstIndex, parentIndex); + parent = worst; + } + } + + @Override + public final void close() { + Releasables.close(values, heapMode); + } +} diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java index 06b1375ac057e..ee747d98c26f8 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperator.java @@ -165,6 +165,7 @@ public int get(int i) { } } success = true; + return page.appendBlocks(blocks); } catch (IOException e) { throw new UncheckedIOException(e); } finally { @@ -172,7 +173,6 @@ public int get(int i) { Releasables.closeExpectNoException(blocks); } } - return page.appendBlocks(blocks); } private void positionFieldWork(int shard, int segment, int firstDoc) { @@ -233,6 +233,7 @@ private void loadFromSingleLeaf(Block[] blocks, int shard, int segment, BlockLoa new RowStrideReaderWork( field.rowStride(ctx), (Block.Builder) field.loader.builder(loaderBlockFactory, docs.count()), + field.loader, f ) ); @@ -262,17 +263,13 @@ private void loadFromSingleLeaf(Block[] blocks, int shard, int segment, BlockLoa ); for (int p = 0; p < docs.count(); p++) { int doc = docs.get(p); - if (storedFields != null) { - storedFields.advanceTo(doc); - } - for (int r = 0; r < rowStrideReaders.size(); r++) { - RowStrideReaderWork work = rowStrideReaders.get(r); - work.reader.read(doc, storedFields, work.builder); + storedFields.advanceTo(doc); + for (RowStrideReaderWork work : rowStrideReaders) { + work.read(doc, storedFields); } } - for (int r = 0; r < rowStrideReaders.size(); r++) { - RowStrideReaderWork work = rowStrideReaders.get(r); - blocks[work.offset] = work.builder.build(); + for (RowStrideReaderWork work : rowStrideReaders) { + blocks[work.offset] = work.build(); } } finally { Releasables.close(rowStrideReaders); @@ -310,7 +307,9 @@ private class LoadFromMany implements Releasable { private final IntVector docs; private final int[] forwards; private final int[] backwards; - private final Block.Builder[] builders; + private final Block.Builder[][] builders; + private final BlockLoader[][] converters; + private final Block.Builder[] fieldTypeBuilders; private final BlockLoader.RowStrideReader[] rowStride; BlockLoaderStoredFieldsFromLeafLoader storedFields; @@ -322,21 +321,25 @@ private class LoadFromMany implements Releasable { docs = docVector.docs(); forwards = docVector.shardSegmentDocMapForwards(); backwards = docVector.shardSegmentDocMapBackwards(); - builders = new Block.Builder[target.length]; + fieldTypeBuilders = new Block.Builder[target.length]; + builders = new Block.Builder[target.length][shardContexts.size()]; + converters = new BlockLoader[target.length][shardContexts.size()]; rowStride = new BlockLoader.RowStrideReader[target.length]; } void run() throws IOException { for (int f = 0; f < fields.length; f++) { /* - * Important note: each block loader has a method to build an - * optimized block loader, but we have *many* fields and some - * of those block loaders may not be compatible with each other. - * So! We take the least common denominator which is the loader - * from the element expected element type. + * Important note: each field has a desired type, which might not match the mapped type (in the case of union-types). + * We create the final block builders using the desired type, one for each field, but then also use inner builders + * (one for each field and shard), and converters (again one for each field and shard) to actually perform the field + * loading in a way that is correct for the mapped field type, and then convert between that type and the desired type. */ - builders[f] = fields[f].info.type.newBlockBuilder(docs.getPositionCount(), blockFactory); + fieldTypeBuilders[f] = fields[f].info.type.newBlockBuilder(docs.getPositionCount(), blockFactory); + builders[f] = new Block.Builder[shardContexts.size()]; + converters[f] = new BlockLoader[shardContexts.size()]; } + ComputeBlockLoaderFactory loaderBlockFactory = new ComputeBlockLoaderFactory(blockFactory, docs.getPositionCount()); int p = forwards[0]; int shard = shards.getInt(p); int segment = segments.getInt(p); @@ -344,7 +347,8 @@ void run() throws IOException { positionFieldWork(shard, segment, firstDoc); LeafReaderContext ctx = ctx(shard, segment); fieldsMoved(ctx, shard); - read(firstDoc); + verifyBuilders(loaderBlockFactory, shard); + read(firstDoc, shard); for (int i = 1; i < forwards.length; i++) { p = forwards[i]; shard = shards.getInt(p); @@ -354,11 +358,19 @@ void run() throws IOException { ctx = ctx(shard, segment); fieldsMoved(ctx, shard); } - read(docs.getInt(p)); + verifyBuilders(loaderBlockFactory, shard); + read(docs.getInt(p), shard); } - for (int f = 0; f < builders.length; f++) { - try (Block orig = builders[f].build()) { - target[f] = orig.filter(backwards); + for (int f = 0; f < target.length; f++) { + for (int s = 0; s < shardContexts.size(); s++) { + if (builders[f][s] != null) { + try (Block orig = (Block) converters[f][s].convert(builders[f][s].build())) { + fieldTypeBuilders[f].copyFrom(orig, 0, orig.getPositionCount()); + } + } + } + try (Block targetBlock = fieldTypeBuilders[f].build()) { + target[f] = targetBlock.filter(backwards); } } } @@ -379,16 +391,29 @@ private void fieldsMoved(LeafReaderContext ctx, int shard) throws IOException { } } - private void read(int doc) throws IOException { + private void verifyBuilders(ComputeBlockLoaderFactory loaderBlockFactory, int shard) { + for (int f = 0; f < fields.length; f++) { + if (builders[f][shard] == null) { + // Note that this relies on field.newShard() to set the loader and converter correctly for the current shard + builders[f][shard] = (Block.Builder) fields[f].loader.builder(loaderBlockFactory, docs.getPositionCount()); + converters[f][shard] = fields[f].loader; + } + } + } + + private void read(int doc, int shard) throws IOException { storedFields.advanceTo(doc); for (int f = 0; f < builders.length; f++) { - rowStride[f].read(doc, storedFields, builders[f]); + rowStride[f].read(doc, storedFields, builders[f][shard]); } } @Override public void close() { - Releasables.closeExpectNoException(builders); + Releasables.closeExpectNoException(fieldTypeBuilders); + for (int f = 0; f < fields.length; f++) { + Releasables.closeExpectNoException(builders[f]); + } } } @@ -468,7 +493,17 @@ private void trackReader(String type, BlockLoader.Reader reader) { } } - private record RowStrideReaderWork(BlockLoader.RowStrideReader reader, Block.Builder builder, int offset) implements Releasable { + private record RowStrideReaderWork(BlockLoader.RowStrideReader reader, Block.Builder builder, BlockLoader loader, int offset) + implements + Releasable { + void read(int doc, BlockLoaderStoredFieldsFromLeafLoader storedFields) throws IOException { + reader.read(doc, storedFields, builder); + } + + Block build() { + return (Block) loader.convert(builder.build()); + } + @Override public void close() { builder.close(); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorFactories.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorFactories.java index 23639109915e2..1e9ea88b2f1d7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorFactories.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/TimeSeriesAggregationOperatorFactories.java @@ -147,7 +147,7 @@ static List valuesAggregatorForGroupings(List new org.elasticsearch.compute.aggregation.ValuesIntAggregatorFunctionSupplier(channels); case LONG -> new org.elasticsearch.compute.aggregation.ValuesLongAggregatorFunctionSupplier(channels); case BOOLEAN -> new org.elasticsearch.compute.aggregation.ValuesBooleanAggregatorFunctionSupplier(channels); - case NULL, DOC, COMPOSITE, UNKNOWN -> throw new IllegalArgumentException("unsupported grouping type"); + case FLOAT, NULL, DOC, COMPOSITE, UNKNOWN -> throw new IllegalArgumentException("unsupported grouping type"); }); aggregators.add(aggregatorSupplier.groupingAggregatorFactory(AggregatorMode.SINGLE)); } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java index dcdba70652910..f647f4fba0225 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/exchange/ExchangeService.java @@ -15,17 +15,19 @@ import org.elasticsearch.action.ActionListenerResponseHandler; import org.elasticsearch.action.support.ChannelActionListener; import org.elasticsearch.common.component.AbstractLifecycleComponent; -import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.concurrent.AbstractAsyncTask; +import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockStreamInput; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; +import org.elasticsearch.tasks.CancellableTask; import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportChannel; @@ -60,6 +62,7 @@ public final class ExchangeService extends AbstractLifecycleComponent { * removed from the exchange service if no sinks are attached (i.e., no computation uses that sink handler). */ public static final String INACTIVE_SINKS_INTERVAL_SETTING = "esql.exchange.sink_inactive_interval"; + public static final TimeValue INACTIVE_SINKS_INTERVAL_DEFAULT = TimeValue.timeValueMinutes(5); private static final Logger LOGGER = LogManager.getLogger(ExchangeService.class); @@ -69,14 +72,17 @@ public final class ExchangeService extends AbstractLifecycleComponent { private final Map sinks = ConcurrentCollections.newConcurrentMap(); - private final InactiveSinksReaper inactiveSinksReaper; - public ExchangeService(Settings settings, ThreadPool threadPool, String executorName, BlockFactory blockFactory) { this.threadPool = threadPool; this.executor = threadPool.executor(executorName); this.blockFactory = blockFactory; - final var inactiveInterval = settings.getAsTime(INACTIVE_SINKS_INTERVAL_SETTING, TimeValue.timeValueMinutes(5)); - this.inactiveSinksReaper = new InactiveSinksReaper(LOGGER, threadPool, this.executor, inactiveInterval); + final var inactiveInterval = settings.getAsTime(INACTIVE_SINKS_INTERVAL_SETTING, INACTIVE_SINKS_INTERVAL_DEFAULT); + // Run the reaper every half of the keep_alive interval + this.threadPool.scheduleWithFixedDelay( + new InactiveSinksReaper(LOGGER, threadPool, inactiveInterval), + TimeValue.timeValueMillis(Math.max(1, inactiveInterval.millis() / 2)), + executor + ); } public void registerTransportHandler(TransportService transportService) { @@ -194,35 +200,56 @@ public void messageReceived(OpenExchangeRequest request, TransportChannel channe private class ExchangeTransportAction implements TransportRequestHandler { @Override - public void messageReceived(ExchangeRequest request, TransportChannel channel, Task task) { + public void messageReceived(ExchangeRequest request, TransportChannel channel, Task exchangeTask) { final String exchangeId = request.exchangeId(); ActionListener listener = new ChannelActionListener<>(channel); final ExchangeSinkHandler sinkHandler = sinks.get(exchangeId); if (sinkHandler == null) { listener.onResponse(new ExchangeResponse(blockFactory, null, true)); } else { + final CancellableTask task = (CancellableTask) exchangeTask; + task.addListener(() -> sinkHandler.onFailure(new TaskCancelledException("request cancelled " + task.getReasonCancelled()))); sinkHandler.fetchPageAsync(request.sourcesFinished(), listener); } } } - private final class InactiveSinksReaper extends AbstractAsyncTask { - InactiveSinksReaper(Logger logger, ThreadPool threadPool, Executor executor, TimeValue interval) { - super(logger, threadPool, executor, interval, true); - rescheduleIfNecessary(); + private final class InactiveSinksReaper extends AbstractRunnable { + private final Logger logger; + private final TimeValue keepAlive; + private final ThreadPool threadPool; + + InactiveSinksReaper(Logger logger, ThreadPool threadPool, TimeValue keepAlive) { + this.logger = logger; + this.keepAlive = keepAlive; + this.threadPool = threadPool; + } + + @Override + public void onFailure(Exception e) { + logger.error("unexpected error when closing inactive sinks", e); + assert false : e; + } + + @Override + public void onRejection(Exception e) { + if (e instanceof EsRejectedExecutionException esre && esre.isExecutorShutdown()) { + logger.debug("rejected execution when closing inactive sinks"); + } else { + onFailure(e); + } } @Override - protected boolean mustReschedule() { - Lifecycle.State state = lifecycleState(); - return state != Lifecycle.State.STOPPED && state != Lifecycle.State.CLOSED; + public boolean isForceExecution() { + // mustn't reject this task even if the queue is full + return true; } @Override - protected void runInternal() { + protected void doRun() { assert Transports.assertNotTransportThread("reaping inactive exchanges can be expensive"); assert ThreadPool.assertNotScheduleThread("reaping inactive exchanges can be expensive"); - final TimeValue maxInterval = getInterval(); final long nowInMillis = threadPool.relativeTimeInMillis(); for (Map.Entry e : sinks.entrySet()) { ExchangeSinkHandler sink = e.getValue(); @@ -230,7 +257,7 @@ protected void runInternal() { continue; } long elapsed = nowInMillis - sink.lastUpdatedTimeInMillis(); - if (elapsed > maxInterval.millis()) { + if (elapsed > keepAlive.millis()) { finishSinkHandler( e.getKey(), new ElasticsearchTimeoutException( @@ -320,7 +347,7 @@ protected void doStart() { @Override protected void doStop() { - inactiveSinksReaper.close(); + } @Override diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoder.java index 6529b79f95295..f1ae4cab8a4bd 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/DefaultUnsortableTopNEncoder.java @@ -21,6 +21,7 @@ final class DefaultUnsortableTopNEncoder implements TopNEncoder { public static final VarHandle LONG = MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.nativeOrder()); public static final VarHandle INT = MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.nativeOrder()); + public static final VarHandle FLOAT = MethodHandles.byteArrayViewVarHandle(float[].class, ByteOrder.nativeOrder()); public static final VarHandle DOUBLE = MethodHandles.byteArrayViewVarHandle(double[].class, ByteOrder.nativeOrder()); @Override @@ -120,6 +121,24 @@ public int decodeInt(BytesRef bytes) { return v; } + @Override + public void encodeFloat(float value, BreakingBytesRefBuilder bytesRefBuilder) { + bytesRefBuilder.grow(bytesRefBuilder.length() + Float.BYTES); + FLOAT.set(bytesRefBuilder.bytes(), bytesRefBuilder.length(), value); + bytesRefBuilder.setLength(bytesRefBuilder.length() + Float.BYTES); + } + + @Override + public float decodeFloat(BytesRef bytes) { + if (bytes.length < Float.BYTES) { + throw new IllegalArgumentException("not enough bytes"); + } + float v = (float) FLOAT.get(bytes.bytes, bytes.offset); + bytes.offset += Float.BYTES; + bytes.length -= Float.BYTES; + return v; + } + @Override public void encodeDouble(double value, BreakingBytesRefBuilder bytesRefBuilder) { bytesRefBuilder.grow(bytesRefBuilder.length() + Double.BYTES); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractor.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractor.java index 0d7d4d476d7b6..b59a3b91e5f39 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractor.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/KeyExtractor.java @@ -12,6 +12,7 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; @@ -31,6 +32,7 @@ static KeyExtractor extractorFor(ElementType elementType, TopNEncoder encoder, b case BYTES_REF -> KeyExtractorForBytesRef.extractorFor(encoder, ascending, nul, nonNul, (BytesRefBlock) block); case INT -> KeyExtractorForInt.extractorFor(encoder, ascending, nul, nonNul, (IntBlock) block); case LONG -> KeyExtractorForLong.extractorFor(encoder, ascending, nul, nonNul, (LongBlock) block); + case FLOAT -> KeyExtractorForFloat.extractorFor(encoder, ascending, nul, nonNul, (FloatBlock) block); case DOUBLE -> KeyExtractorForDouble.extractorFor(encoder, ascending, nul, nonNul, (DoubleBlock) block); case NULL -> new KeyExtractorForNull(nul); default -> { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ResultBuilder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ResultBuilder.java index bd2027cade78f..61c49bac7505d 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ResultBuilder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ResultBuilder.java @@ -50,6 +50,7 @@ static ResultBuilder resultBuilderFor( case BYTES_REF -> new ResultBuilderForBytesRef(blockFactory, encoder, inKey, positions); case INT -> new ResultBuilderForInt(blockFactory, encoder, inKey, positions); case LONG -> new ResultBuilderForLong(blockFactory, encoder, inKey, positions); + case FLOAT -> new ResultBuilderForFloat(blockFactory, encoder, inKey, positions); case DOUBLE -> new ResultBuilderForDouble(blockFactory, encoder, inKey, positions); case NULL -> new ResultBuilderForNull(blockFactory); case DOC -> new ResultBuilderForDoc(blockFactory, positions); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/SortableTopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/SortableTopNEncoder.java index d04064e0a6777..6ba653c3adedf 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/SortableTopNEncoder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/SortableTopNEncoder.java @@ -51,6 +51,24 @@ public final int decodeInt(BytesRef bytes) { return v; } + @Override + public final void encodeFloat(float value, BreakingBytesRefBuilder bytesRefBuilder) { + bytesRefBuilder.grow(bytesRefBuilder.length() + Integer.BYTES); + NumericUtils.intToSortableBytes(NumericUtils.floatToSortableInt(value), bytesRefBuilder.bytes(), bytesRefBuilder.length()); + bytesRefBuilder.setLength(bytesRefBuilder.length() + Integer.BYTES); + } + + @Override + public final float decodeFloat(BytesRef bytes) { + if (bytes.length < Float.BYTES) { + throw new IllegalArgumentException("not enough bytes"); + } + float v = NumericUtils.sortableIntToFloat(NumericUtils.sortableBytesToInt(bytes.bytes, bytes.offset)); + bytes.offset += Float.BYTES; + bytes.length -= Float.BYTES; + return v; + } + @Override public final void encodeDouble(double value, BreakingBytesRefBuilder bytesRefBuilder) { bytesRefBuilder.grow(bytesRefBuilder.length() + Long.BYTES); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java index f1fb7cb7736c5..737a602543db7 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/TopNEncoder.java @@ -54,6 +54,10 @@ public interface TopNEncoder { int decodeInt(BytesRef bytes); + void encodeFloat(float value, BreakingBytesRefBuilder bytesRefBuilder); + + float decodeFloat(BytesRef bytes); + void encodeDouble(double value, BreakingBytesRefBuilder bytesRefBuilder); double decodeDouble(BytesRef bytes); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java index af870dd336a74..b9336024eb404 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/topn/ValueExtractor.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.data.DocBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; @@ -32,6 +33,7 @@ static ValueExtractor extractorFor(ElementType elementType, TopNEncoder encoder, case BYTES_REF -> ValueExtractorForBytesRef.extractorFor(encoder, inKey, (BytesRefBlock) block); case INT -> ValueExtractorForInt.extractorFor(encoder, inKey, (IntBlock) block); case LONG -> ValueExtractorForLong.extractorFor(encoder, inKey, (LongBlock) block); + case FLOAT -> ValueExtractorForFloat.extractorFor(encoder, inKey, (FloatBlock) block); case DOUBLE -> ValueExtractorForDouble.extractorFor(encoder, inKey, (DoubleBlock) block); case NULL -> new ValueExtractorForNull(); case DOC -> new ValueExtractorForDoc(encoder, ((DocBlock) block).asVector()); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java index 08e8bca64bbe7..6e56b96bda06e 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/AggregatorFunctionTestCase.java @@ -16,6 +16,7 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -223,6 +224,11 @@ protected static Stream allBooleans(Block input) { return allValueOffsets(b).mapToObj(i -> b.getBoolean(i)); } + protected static Stream allFloats(Block input) { + FloatBlock b = (FloatBlock) input; + return allValueOffsets(b).mapToObj(b::getFloat); + } + protected static DoubleStream allDoubles(Block input) { DoubleBlock b = (DoubleBlock) input; return allValueOffsets(b).mapToDouble(i -> b.getDouble(i)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..7f520de393d73 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatAggregatorFunctionTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.BasicBlockTests; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class CountDistinctFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new CountDistinctFloatAggregatorFunctionSupplier(inputChannels, 40000); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "count_distinct of floats"; + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + long expected = input.stream().flatMap(AggregatorFunctionTestCase::allFloats).distinct().count(); + + long count = ((LongBlock) result).getLong(0); + // HLL is an approximation algorithm and precision depends on the number of values computed and the precision_threshold param + // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-cardinality-aggregation.html + // For a number of values close to 10k and precision_threshold=1000, precision should be less than 10% + assertThat((double) count, closeTo(expected, expected * .1)); + } + + @Override + protected void assertOutputFromEmpty(Block b) { + assertThat(b.getPositionCount(), equalTo(1)); + assertThat(BasicBlockTests.valuesAtPositions(b, 0, 1), equalTo(List.of(List.of(0L)))); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..03a11bb976b21 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/CountDistinctFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class CountDistinctFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new CountDistinctFloatAggregatorFunctionSupplier(inputChannels, 40000); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "count_distinct of floats"; + } + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, size).mapToObj(l -> Tuple.tuple(randomGroupId(size), randomFloatBetween(0, 100, true))) + ); + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + long distinct = input.stream().flatMap(p -> allFloats(p, group)).distinct().count(); + long count = ((LongBlock) result).getLong(position); + // HLL is an approximation algorithm and precision depends on the number of values computed and the precision_threshold param + // https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-cardinality-aggregation.html + // For a number of values close to 10k and precision_threshold=1000, precision should be less than 10% + assertThat((double) count, closeTo(distinct, distinct * 0.1)); + } + + @Override + protected void assertOutputFromNullOnly(Block b, int position) { + assertThat(b.isNull(position), equalTo(false)); + assertThat(b.getValueCount(position), equalTo(1)); + assertThat(((LongBlock) b).getLong(b.getFirstValueIndex(position)), equalTo(0L)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java index d10e1bada5580..3436d6b537611 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/GroupingAggregatorFunctionTestCase.java @@ -17,6 +17,7 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.IntVector; import org.elasticsearch.compute.data.LongBlock; @@ -202,6 +203,7 @@ protected void appendNull(ElementType elementType, Block.Builder builder, int bl append(builder, switch (elementType) { case BOOLEAN -> randomBoolean(); case BYTES_REF -> new BytesRef(randomAlphaOfLength(3)); + case FLOAT -> randomFloat(); case DOUBLE -> randomDouble(); case INT -> 1; case LONG -> 1L; @@ -276,7 +278,7 @@ public final void testMulitvaluedNullGroupsAndValues() { assertSimpleOutput(origInput, results); } - public void testMulitvaluedNullGroup() { + public final void testMulitvaluedNullGroup() { DriverContext driverContext = driverContext(); BlockFactory blockFactory = driverContext.blockFactory(); int end = between(1, 2); // TODO revert @@ -479,6 +481,11 @@ protected static Stream allBooleans(Page page, Long group) { return allValueOffsets(page, group).mapToObj(i -> b.getBoolean(i)); } + protected static Stream allFloats(Page page, Long group) { + FloatBlock b = page.getBlock(1); + return allValueOffsets(page, group).mapToObj(b::getFloat); + } + protected static DoubleStream allDoubles(Page page, Long group) { DoubleBlock b = page.getBlock(1); return allValueOffsets(page, group).mapToDouble(i -> b.getDouble(i)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..5e14a99fd0fa2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatAggregatorFunctionTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MaxFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MaxFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "max of floats"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Float max = input.stream().flatMap(AggregatorFunctionTestCase::allFloats).max(floatComparator()).get(); + assertThat(((FloatBlock) result).getFloat(0), equalTo(max)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..0abcb05a91af6 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MaxFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.Optional; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MaxFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, end).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MaxFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "max of floats"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Optional max = input.stream().flatMap(p -> allFloats(p, group)).max(floatComparator()); + if (max.isEmpty()) { + assertThat(result.isNull(position), equalTo(true)); + return; + } + assertThat(result.isNull(position), equalTo(false)); + assertThat(((FloatBlock) result).getFloat(position), equalTo(max.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests.java index 8eba1842d688d..a6ca769036e54 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationDoubleGroupingAggregatorFunctionTests.java @@ -76,9 +76,4 @@ static double median(DoubleStream s) { int c = data.length / 2; return data.length % 2 == 0 ? (data[c - 1] + data[c]) / 2 : data[c]; } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/101569") - public void testMulitvaluedNullGroup() { - // only here for muting it - } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..786603e12f9c8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatAggregatorFunctionTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.Randomness; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.Matchers.closeTo; + +public class MedianAbsoluteDeviationFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + List values = Arrays.asList(1.2f, 1.25f, 2.0f, 2.0f, 4.3f, 6.0f, 9.0f); + Randomness.shuffle(values); + return new SequenceFloatBlockSourceOperator(blockFactory, values.subList(0, Math.min(values.size(), end))); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MedianAbsoluteDeviationFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "median_absolute_deviation of floats"; + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + assertThat(((DoubleBlock) result).getDouble(0), closeTo(0.8, 0.001d)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..14416b3aec1ee --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.common.Randomness; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.closeTo; + +public class MedianAbsoluteDeviationFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + Float[][] samples = new Float[][] { + { 1.2f, 1.25f, 2.0f, 2.0f, 4.3f, 6.0f, 9.0f }, + { 0.1f, 1.5f, 2.0f, 3.0f, 4.0f, 7.5f, 100.0f }, + { 0.2f, 1.75f, 2.0f, 2.5f }, + { 0.5f, 3.0f, 3.0f, 3.0f, 4.3f }, + { 0.25f, 1.5f, 3.0f } }; + List> values = new ArrayList<>(); + for (int i = 0; i < samples.length; i++) { + List list = Arrays.stream(samples[i]).collect(Collectors.toList()); + Randomness.shuffle(list); + for (float v : list) { + values.add(Tuple.tuple((long) i, v)); + } + } + return new LongFloatTupleBlockSourceOperator(blockFactory, values.subList(0, Math.min(values.size(), end))); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MedianAbsoluteDeviationFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "median_absolute_deviation of floats"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + double medianAbsoluteDeviation = medianAbsoluteDeviation(input.stream().flatMap(p -> allFloats(p, group))); + assertThat(((DoubleBlock) result).getDouble(position), closeTo(medianAbsoluteDeviation, medianAbsoluteDeviation * .000001)); + } + + static double medianAbsoluteDeviation(Stream s) { + Float[] data = s.toArray(Float[]::new); + float median = median(Arrays.stream(data)); + return median(Arrays.stream(data).map(d -> Math.abs(median - d))); + } + + static float median(Stream s) { + // The input data is small enough that tdigest will find the actual median. + Float[] data = s.sorted().toArray(Float[]::new); + if (data.length == 0) { + return 0; + } + int c = data.length / 2; + return data.length % 2 == 0 ? (data[c - 1] + data[c]) / 2 : data[c]; + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..59a09569c65a2 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatAggregatorFunctionTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MinFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MinFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "min of floats"; + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + Float min = input.stream().flatMap(b -> allFloats(b)).min(floatComparator()).get(); + assertThat(((FloatBlock) result).getFloat(0), equalTo(min)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..be41e058f60da --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/MinFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.Optional; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.equalTo; + +public class MinFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, end).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new MinFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "min of floats"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + Optional min = input.stream().flatMap(p -> allFloats(p, group)).min(floatComparator()); + if (min.isEmpty()) { + assertThat(result.isNull(position), equalTo(true)); + return; + } + assertThat(result.isNull(position), equalTo(false)); + assertThat(((FloatBlock) result).getFloat(position), equalTo(min.get())); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..6e4a1e09640dc --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatAggregatorFunctionTests.java @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.elasticsearch.test.ESTestCase; +import org.junit.Before; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; + +public class PercentileFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + + private double percentile; + + @Before + public void initParameters() { + percentile = randomFrom(0, 1, 5, 10, 25, 50, 75, 90, 95, 99, 100); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new PercentileFloatAggregatorFunctionSupplier(inputChannels, percentile); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "percentile of floats"; + } + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + TDigestState td = TDigestState.create(QuantileStates.DEFAULT_COMPRESSION); + input.stream().flatMap(AggregatorFunctionTestCase::allFloats).forEach(td::add); + double expected = td.quantile(percentile / 100); + double value = ((DoubleBlock) result).getDouble(0); + assertThat(value, closeTo(expected, expected * 0.1)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..84a97a7cd30ac --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/PercentileFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.aggregations.metrics.TDigestState; +import org.junit.Before; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; + +public class PercentileFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + + private double percentile; + + @Before + public void initParameters() { + percentile = randomFrom(0, 1, 5, 10, 25, 50, 75, 90, 95, 99, 100); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new PercentileFloatAggregatorFunctionSupplier(inputChannels, percentile); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "percentile of floats"; + } + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, end).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + TDigestState td = TDigestState.create(QuantileStates.DEFAULT_COMPRESSION); + input.stream().flatMap(p -> allFloats(p, group)).forEach(td::add); + if (td.size() > 0) { + double expected = td.quantile(percentile / 100); + double value = ((DoubleBlock) result).getDouble(position); + assertThat(value, closeTo(expected, expected * 0.1)); + } else { + assertTrue(result.isNull(position)); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..d365f02d289c8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatAggregatorFunctionTests.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.Driver; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.compute.operator.TestResultPageSinkOperator; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.LongStream; +import java.util.stream.Stream; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class SumFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, LongStream.range(0, size).mapToObj(l -> ESTestCase.randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new SumFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "sum of floats"; + } + + @Override + protected void assertSimpleOutput(List input, Block result) { + double sum = input.stream().flatMap(AggregatorFunctionTestCase::allFloats).mapToDouble(f -> (double) f).sum(); + assertThat(((DoubleBlock) result).getDouble(0), closeTo(sum, .0001)); + } + + public void testOverflowSucceeds() { + DriverContext driverContext = driverContext(); + List results = new ArrayList<>(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator(driverContext.blockFactory(), Stream.of(Float.MAX_VALUE - 1, 2f)), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertThat(results.get(0).getBlock(0).getDouble(0), equalTo((double) Float.MAX_VALUE + 1)); + assertDriverContext(driverContext); + } + + public void testSummationAccuracy() { + DriverContext driverContext = driverContext(); + List results = new ArrayList<>(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator( + driverContext.blockFactory(), + Stream.of(0.1f, 0.2f, 0.3f, 0.4f, 0.5f, 0.6f, 0.7f, 0.8f, 0.9f, 1.0f, 1.1f, 1.2f, 1.3f, 1.4f, 1.5f, 1.6f, 1.7f) + ), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertEquals(15.3, results.get(0).getBlock(0).getDouble(0), 0.001); + assertDriverContext(driverContext); + + // Summing up an array which contains NaN and infinities and expect a result same as naive summation + results.clear(); + int n = randomIntBetween(5, 10); + Float[] values = new Float[n]; + float sum = 0; + for (int i = 0; i < n; i++) { + values[i] = frequently() + ? randomFrom(Float.NaN, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY) + : randomFloatBetween(Float.MIN_VALUE, Float.MAX_VALUE, true); + sum += values[i]; + } + driverContext = driverContext(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator(driverContext.blockFactory(), Stream.of(values)), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertEquals(sum, results.get(0).getBlock(0).getDouble(0), 1e-10); + assertDriverContext(driverContext); + + // Summing up some big float values and expect a big double result + results.clear(); + n = randomIntBetween(5, 10); + Float[] largeValues = new Float[n]; + for (int i = 0; i < n; i++) { + largeValues[i] = Float.MAX_VALUE; + } + driverContext = driverContext(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator(driverContext.blockFactory(), Stream.of(largeValues)), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertEquals((double) Float.MAX_VALUE * n, results.get(0).getBlock(0).getDouble(0), 0d); + assertDriverContext(driverContext); + + results.clear(); + for (int i = 0; i < n; i++) { + largeValues[i] = -Float.MAX_VALUE; + } + driverContext = driverContext(); + try ( + Driver d = new Driver( + driverContext, + new SequenceFloatBlockSourceOperator(driverContext.blockFactory(), Stream.of(largeValues)), + List.of(simple().get(driverContext)), + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + } + assertEquals((double) -Float.MAX_VALUE * n, results.get(0).getBlock(0).getDouble(0), 0d); + assertDriverContext(driverContext); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..54bd92cbfff21 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/SumFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.search.aggregations.metrics.CompensatedSum; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.closeTo; + +public class SumFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int end) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, end).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new SumFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "sum of floats"; + } + + @Override + protected void assertSimpleGroup(List input, Block result, int position, Long group) { + CompensatedSum sum = new CompensatedSum(); + input.stream().flatMap(p -> allFloats(p, group)).mapToDouble(f -> (double) f).forEach(sum::add); + // Won't precisely match in distributed case but will be close + assertThat(((DoubleBlock) result).getDouble(position), closeTo(sum.value(), 0.01)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunctionTests.java new file mode 100644 index 0000000000000..f708038776032 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListDoubleAggregatorFunctionTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceDoubleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.contains; + +public class TopListDoubleAggregatorFunctionTests extends AggregatorFunctionTestCase { + private static final int LIMIT = 100; + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceDoubleBlockSourceOperator(blockFactory, IntStream.range(0, size).mapToDouble(l -> randomDouble())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new TopListDoubleAggregatorFunctionSupplier(inputChannels, LIMIT, true); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "top_list of doubles"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMapToDouble(b -> allDoubles(b)).sorted().limit(LIMIT).boxed().toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), contains(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..98a016783955e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListFloatAggregatorFunctionTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.contains; + +public class TopListFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + private static final int LIMIT = 100; + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, IntStream.range(0, size).mapToObj(l -> randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new TopListFloatAggregatorFunctionSupplier(inputChannels, LIMIT, true); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "top_list of floats"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMap(b -> allFloats(b)).sorted().limit(LIMIT).toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), contains(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunctionTests.java new file mode 100644 index 0000000000000..443604efd5c15 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListIntAggregatorFunctionTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceIntBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.contains; + +public class TopListIntAggregatorFunctionTests extends AggregatorFunctionTestCase { + private static final int LIMIT = 100; + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceIntBlockSourceOperator(blockFactory, IntStream.range(0, size).map(l -> randomInt())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new TopListIntAggregatorFunctionSupplier(inputChannels, LIMIT, true); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "top_list of ints"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMapToInt(b -> allInts(b)).sorted().limit(LIMIT).boxed().toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), contains(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunctionTests.java new file mode 100644 index 0000000000000..4a6f101e573b8 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/TopListLongAggregatorFunctionTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceLongBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.contains; + +public class TopListLongAggregatorFunctionTests extends AggregatorFunctionTestCase { + private static final int LIMIT = 100; + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceLongBlockSourceOperator(blockFactory, LongStream.range(0, size).map(l -> randomLong())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new TopListLongAggregatorFunctionSupplier(inputChannels, LIMIT, true); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "top_list of longs"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMapToLong(b -> allLongs(b)).sorted().limit(LIMIT).boxed().toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), contains(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionTests.java new file mode 100644 index 0000000000000..899a89dd993a4 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatAggregatorFunctionTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.operator.SequenceFloatBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.hamcrest.Matchers.containsInAnyOrder; + +public class ValuesFloatAggregatorFunctionTests extends AggregatorFunctionTestCase { + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new SequenceFloatBlockSourceOperator(blockFactory, IntStream.range(0, size).mapToObj(i -> randomFloat())); + } + + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new ValuesFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "values of floats"; + } + + @Override + public void assertSimpleOutput(List input, Block result) { + Object[] values = input.stream().flatMap(AggregatorFunctionTestCase::allFloats).collect(Collectors.toSet()).toArray(Object[]::new); + assertThat((List) BlockUtils.toJavaObject(result, 0), containsInAnyOrder(values)); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunctionTests.java new file mode 100644 index 0000000000000..787b6fd4c75be --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesFloatGroupingAggregatorFunctionTests.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.aggregation; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.operator.LongFloatTupleBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.nullValue; + +public class ValuesFloatGroupingAggregatorFunctionTests extends GroupingAggregatorFunctionTestCase { + @Override + protected AggregatorFunctionSupplier aggregatorFunction(List inputChannels) { + return new ValuesFloatAggregatorFunctionSupplier(inputChannels); + } + + @Override + protected String expectedDescriptionOfAggregator() { + return "values of floats"; + } + + @Override + protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { + return new LongFloatTupleBlockSourceOperator( + blockFactory, + LongStream.range(0, size).mapToObj(l -> Tuple.tuple(randomLongBetween(0, 4), randomFloat())) + ); + } + + @Override + public void assertSimpleGroup(List input, Block result, int position, Long group) { + Object[] values = input.stream().flatMap(p -> allFloats(p, group)).collect(Collectors.toSet()).toArray(Object[]::new); + Object resultValue = BlockUtils.toJavaObject(result, position); + switch (values.length) { + case 0 -> assertThat(resultValue, nullValue()); + case 1 -> assertThat(resultValue, equalTo(values[0])); + default -> assertThat((List) resultValue, containsInAnyOrder(values)); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesIntAggregatorFunctionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesIntAggregatorFunctionTests.java index 9d421c7801a43..46e31b589997a 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesIntAggregatorFunctionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/ValuesIntAggregatorFunctionTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.compute.aggregation; +import org.apache.lucene.tests.util.LuceneTestCase; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BlockUtils; @@ -19,6 +20,7 @@ import static org.hamcrest.Matchers.containsInAnyOrder; +@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/109932") public class ValuesIntAggregatorFunctionTests extends AggregatorFunctionTestCase { @Override protected SourceOperator simpleInput(BlockFactory blockFactory, int size) { diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java index ac0ad5f9fbd99..81c32670289c2 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicBlockTests.java @@ -78,6 +78,10 @@ void testEmpty(BlockFactory bf) { assertZeroPositionsAndRelease(bf.newLongBlockBuilder(0).build()); assertZeroPositionsAndRelease(bf.newLongArrayVector(new long[] {}, 0)); assertZeroPositionsAndRelease(bf.newLongVectorBuilder(0).build()); + assertZeroPositionsAndRelease(bf.newFloatArrayBlock(new float[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); + assertZeroPositionsAndRelease(bf.newFloatBlockBuilder(0).build()); + assertZeroPositionsAndRelease(bf.newFloatArrayVector(new float[] {}, 0)); + assertZeroPositionsAndRelease(bf.newFloatVectorBuilder(0).build()); assertZeroPositionsAndRelease(bf.newDoubleArrayBlock(new double[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering())); assertZeroPositionsAndRelease(bf.newDoubleBlockBuilder(0).build()); assertZeroPositionsAndRelease(bf.newDoubleArrayVector(new double[] {}, 0)); @@ -116,6 +120,17 @@ public void testSmallSingleValueDenseGrowthLong() { } } + public void testSmallSingleValueDenseGrowthFloat() { + for (int initialSize : List.of(0, 1, 2, 3, 4, 5)) { + try (var blockBuilder = blockFactory.newFloatBlockBuilder(initialSize)) { + IntStream.range(0, 10).forEach(blockBuilder::appendFloat); + FloatBlock block = blockBuilder.build(); + assertSingleValueDenseBlock(block); + block.close(); + } + } + } + public void testSmallSingleValueDenseGrowthDouble() { for (int initialSize : List.of(0, 1, 2, 3, 4, 5)) { try (var blockBuilder = blockFactory.newDoubleBlockBuilder(initialSize)) { @@ -487,6 +502,100 @@ public void testConstantDoubleBlock() { } } + public void testFloatBlock() { + for (int i = 0; i < 1000; i++) { + assertThat(breaker.getUsed(), is(0L)); + int positionCount = randomIntBetween(1, 16 * 1024); + FloatBlock block; + if (randomBoolean()) { + final int builderEstimateSize = randomBoolean() ? randomIntBetween(1, positionCount) : positionCount; + var blockBuilder = blockFactory.newFloatBlockBuilder(builderEstimateSize); + IntStream.range(0, positionCount).forEach(blockBuilder::appendFloat); + block = blockBuilder.build(); + } else { + float[] fa = new float[positionCount]; + IntStream.range(0, positionCount).forEach(v -> fa[v] = (float) v); + block = blockFactory.newFloatArrayVector(fa, positionCount).asBlock(); + } + + assertThat(positionCount, is(block.getPositionCount())); + assertThat(0f, is(block.getFloat(0))); + assertThat((float) positionCount - 1, is(block.getFloat(positionCount - 1))); + int pos = (int) block.getFloat(randomPosition(positionCount)); + assertThat((float) pos, is(block.getFloat(pos))); + assertSingleValueDenseBlock(block); + if (positionCount > 2) { + assertLookup(block, positions(blockFactory, 1, 2, new int[] { 1, 2 }), List.of(List.of(1f), List.of(2f), List.of(1f, 2f))); + } + assertLookup(block, positions(blockFactory, positionCount + 1000), singletonList(null)); + assertEmptyLookup(blockFactory, block); + + try (FloatBlock.Builder blockBuilder = blockFactory.newFloatBlockBuilder(1)) { + FloatBlock copy = blockBuilder.copyFrom(block, 0, block.getPositionCount()).build(); + assertThat(copy, equalTo(block)); + releaseAndAssertBreaker(block, copy); + } + + if (positionCount > 1) { + assertNullValues( + positionCount, + blockFactory::newFloatBlockBuilder, + FloatBlock.Builder::appendFloat, + position -> (float) position, + FloatBlock.Builder::build, + (randomNonNullPosition, b) -> { + assertThat((float) randomNonNullPosition, is(b.getFloat(randomNonNullPosition.intValue()))); + } + ); + } + + try ( + DoubleVector.Builder vectorBuilder = blockFactory.newDoubleVectorBuilder( + randomBoolean() ? randomIntBetween(1, positionCount) : positionCount + ) + ) { + IntStream.range(0, positionCount).mapToDouble(ii -> 1.0 / ii).forEach(vectorBuilder::appendDouble); + DoubleVector vector = vectorBuilder.build(); + assertSingleValueDenseBlock(vector.asBlock()); + releaseAndAssertBreaker(vector.asBlock()); + } + } + } + + public void testConstantFloatBlock() { + for (int i = 0; i < 1000; i++) { + int positionCount = randomIntBetween(1, 16 * 1024); + float value = randomFloat(); + FloatBlock block = blockFactory.newConstantFloatBlockWith(value, positionCount); + assertThat(positionCount, is(block.getPositionCount())); + assertThat(value, is(block.getFloat(0))); + assertThat(value, is(block.getFloat(positionCount - 1))); + assertThat(value, is(block.getFloat(randomPosition(positionCount)))); + assertSingleValueDenseBlock(block); + if (positionCount > 2) { + assertLookup( + block, + positions(blockFactory, 1, 2, new int[] { 1, 2 }), + List.of(List.of(value), List.of(value), List.of(value, value)) + ); + assertLookup( + block, + positions(blockFactory, 1, 2), + List.of(List.of(value), List.of(value)), + b -> assertThat(b.asVector(), instanceOf(ConstantFloatVector.class)) + ); + } + assertLookup( + block, + positions(blockFactory, positionCount + 1000), + singletonList(null), + b -> assertThat(b, instanceOf(ConstantNullBlock.class)) + ); + assertEmptyLookup(blockFactory, block); + releaseAndAssertBreaker(block); + } + } + private void testBytesRefBlock(Supplier byteArraySupplier, boolean chomp, org.mockito.ThrowingConsumer assertions) { int positionCount = randomIntBetween(1, 16 * 1024); BytesRef[] values = new BytesRef[positionCount]; @@ -1030,6 +1139,7 @@ public static List> valuesAtPositions(Block block, int from, int to positionValues.add(switch (block.elementType()) { case INT -> ((IntBlock) block).getInt(i++); case LONG -> ((LongBlock) block).getLong(i++); + case FLOAT -> ((FloatBlock) block).getFloat(i++); case DOUBLE -> ((DoubleBlock) block).getDouble(i++); case BYTES_REF -> ((BytesRefBlock) block).getBytesRef(i++, new BytesRef()); case BOOLEAN -> ((BooleanBlock) block).getBoolean(i++); @@ -1149,6 +1259,11 @@ public static RandomBlock randomBlock( valuesAtPosition.add(l); ((LongBlock.Builder) builder).appendLong(l); } + case FLOAT -> { + float f = randomFloat(); + valuesAtPosition.add(f); + ((FloatBlock.Builder) builder).appendFloat(f); + } case DOUBLE -> { double d = randomDouble(); valuesAtPosition.add(d); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java index f76ff0708120b..e0cf277e99967 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BasicPageTests.java @@ -136,14 +136,15 @@ public void testEqualityAndHashCode() throws IOException { int blockCount = randomIntBetween(1, 256); Block[] blocks = new Block[blockCount]; for (int blockIndex = 0; blockIndex < blockCount; blockIndex++) { - blocks[blockIndex] = switch (randomInt(6)) { + blocks[blockIndex] = switch (randomInt(7)) { case 0 -> blockFactory.newIntArrayVector(randomInts(positions).toArray(), positions).asBlock(); case 1 -> blockFactory.newLongArrayVector(randomLongs(positions).toArray(), positions).asBlock(); - case 2 -> blockFactory.newDoubleArrayVector(randomDoubles(positions).toArray(), positions).asBlock(); - case 3 -> blockFactory.newConstantIntBlockWith(randomInt(), positions); - case 4 -> blockFactory.newConstantLongBlockWith(randomLong(), positions); - case 5 -> blockFactory.newConstantDoubleBlockWith(randomDouble(), positions); - case 6 -> blockFactory.newConstantBytesRefBlockWith(new BytesRef(Integer.toHexString(randomInt())), positions); + case 2 -> blockFactory.newFloatArrayVector(randomFloats(positions), positions).asBlock(); + case 3 -> blockFactory.newDoubleArrayVector(randomDoubles(positions).toArray(), positions).asBlock(); + case 4 -> blockFactory.newConstantIntBlockWith(randomInt(), positions); + case 5 -> blockFactory.newConstantLongBlockWith(randomLong(), positions); + case 6 -> blockFactory.newConstantDoubleBlockWith(randomDouble(), positions); + case 7 -> blockFactory.newConstantBytesRefBlockWith(new BytesRef(Integer.toHexString(randomInt())), positions); default -> throw new AssertionError(); }; } @@ -184,6 +185,7 @@ public void testPageSerializationSimple() throws IOException { Page origPage = new Page( blockFactory.newIntArrayVector(IntStream.range(0, 10).toArray(), 10).asBlock(), blockFactory.newLongArrayVector(LongStream.range(10, 20).toArray(), 10).asBlock(), + blockFactory.newFloatArrayVector(randomFloats(10), 10).asBlock(), blockFactory.newDoubleArrayVector(LongStream.range(30, 40).mapToDouble(i -> i).toArray(), 10).asBlock(), blockFactory.newBytesRefArrayVector(bytesRefArrayOf("0a", "1b", "2c", "3d", "4e", "5f", "6g", "7h", "8i", "9j"), 10).asBlock(), blockFactory.newConstantIntBlockWith(randomInt(), 10), @@ -248,4 +250,10 @@ BytesRefArray bytesRefArrayOf(String... values) { Arrays.stream(values).map(BytesRef::new).forEach(array::append); return array; } + + float[] randomFloats(int size) { + float[] fa = new float[size]; + IntStream.range(0, size).forEach(i -> fa[i] = randomFloat()); + return fa; + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java index ec9bea2edcb75..5d5eef1fe3c07 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockFactoryTests.java @@ -373,6 +373,96 @@ public void testDoubleVectorBuilderWithPossiblyLargeEstimateRandom() { } } + public void testFloatBlockBuilderWithPossiblyLargeEstimateEmpty() { + var builder = blockFactory.newFloatBlockBuilder(randomIntBetween(0, 2048)); + assertThat(breaker.getUsed(), greaterThan(0L)); + var block = builder.build(); + releaseAndAssertBreaker(block); + + block = blockFactory.newFloatArrayBlock(new float[] {}, 0, new int[] { 0 }, new BitSet(), randomOrdering()); + assertThat(breaker.getUsed(), greaterThan(0L)); + releaseAndAssertBreaker(block); + } + + public void testFloatBlockBuilderWithPossiblyLargeEstimateSingle() { + var builder = blockFactory.newFloatBlockBuilder(randomIntBetween(0, 2048)); + builder.appendFloat(randomFloat()); + assertThat(breaker.getUsed(), greaterThan(0L)); + var block = builder.build(); + releaseAndAssertBreaker(block); + + block = blockFactory.newFloatArrayBlock(new float[] { randomFloat() }, 1, new int[] { 0, 1 }, new BitSet(), randomOrdering()); + assertThat(breaker.getUsed(), greaterThan(0L)); + releaseAndAssertBreaker(block); + + block = blockFactory.newConstantFloatBlockWith(randomFloat(), randomIntBetween(1, 2048)); + assertThat(breaker.getUsed(), greaterThan(0L)); + releaseAndAssertBreaker(block); + } + + public void testFloatBlockBuilderWithPossiblyLargeEstimateRandom() { + for (int i = 0; i < 1000; i++) { + assertThat(breaker.getUsed(), is(0L)); + var builder = blockFactory.newFloatBlockBuilder(randomIntBetween(0, 2048)); + + builder.appendFloat(randomFloat()); + if (randomBoolean()) { // null-ness + builder.appendNull(); + } + if (randomBoolean()) { // mv-ness + builder.beginPositionEntry(); + builder.appendFloat(randomFloat()); + builder.appendFloat(randomFloat()); + builder.endPositionEntry(); + } + builder.appendFloat(randomFloat()); + assertThat(breaker.getUsed(), greaterThan(0L)); + var block = builder.build(); + releaseAndAssertBreaker(block); + } + } + + public void testFloatVectorBuilderWithPossiblyLargeEstimateEmpty() { + var builder = blockFactory.newFloatVectorBuilder(randomIntBetween(0, 2048)); + assertThat(breaker.getUsed(), greaterThan(0L)); + var vector = builder.build(); + releaseAndAssertBreaker(vector); + + vector = blockFactory.newFloatArrayVector(new float[] {}, 0); + assertThat(breaker.getUsed(), greaterThan(0L)); + releaseAndAssertBreaker(vector); + } + + public void testFloatVectorBuilderWithPossiblyLargeEstimateSingle() { + var builder = blockFactory.newFloatVectorBuilder(randomIntBetween(0, 2048)); + builder.appendFloat(randomFloat()); + assertThat(breaker.getUsed(), greaterThan(0L)); + var vector = builder.build(); + releaseAndAssertBreaker(vector); + + vector = blockFactory.newFloatArrayVector(new float[] { randomFloat() }, 1); + assertThat(breaker.getUsed(), greaterThan(0L)); + releaseAndAssertBreaker(vector); + + vector = blockFactory.newConstantFloatBlockWith(randomFloat(), randomIntBetween(1, 2048)).asVector(); + assertThat(breaker.getUsed(), greaterThan(0L)); + releaseAndAssertBreaker(vector); + } + + public void testFloatVectorBuilderWithPossiblyLargeEstimateRandom() { + for (int i = 0; i < 1000; i++) { + assertThat(breaker.getUsed(), is(0L)); + var builder = blockFactory.newFloatVectorBuilder(randomIntBetween(0, 2048)); + builder.appendFloat(randomFloat()); + if (randomBoolean()) { // constant-ness or not + builder.appendFloat(randomFloat()); + } + assertThat(breaker.getUsed(), greaterThan(0L)); + var vector = builder.build(); + releaseAndAssertBreaker(vector); + } + } + public void testBooleanBlockBuilderWithPossiblyLargeEstimateEmpty() { var builder = blockFactory.newBooleanBlockBuilder(randomIntBetween(0, 2048)); assertThat(breaker.getUsed(), greaterThan(0L)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockSerializationTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockSerializationTests.java index 2daf7755841f7..8ca02b64f01ff 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockSerializationTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockSerializationTests.java @@ -39,6 +39,10 @@ public void testConstantLongBlockLong() throws IOException { assertConstantBlockImpl(blockFactory.newConstantLongBlockWith(randomLong(), randomIntBetween(1, 8192))); } + public void testConstantFloatBlock() throws IOException { + assertConstantBlockImpl(blockFactory.newConstantFloatBlockWith(randomFloat(), randomIntBetween(1, 8192))); + } + public void testConstantDoubleBlock() throws IOException { assertConstantBlockImpl(blockFactory.newConstantDoubleBlockWith(randomDouble(), randomIntBetween(1, 8192))); } @@ -81,6 +85,17 @@ public void testEmptyLongBlock() throws IOException { } } + public void testEmptyFloatBlock() throws IOException { + assertEmptyBlock(blockFactory.newFloatBlockBuilder(0).build()); + try (FloatBlock toFilter = blockFactory.newFloatBlockBuilder(0).appendNull().build()) { + assertEmptyBlock(toFilter.filter()); + } + assertEmptyBlock(blockFactory.newFloatVectorBuilder(0).build().asBlock()); + try (FloatVector toFilter = blockFactory.newFloatVectorBuilder(0).appendFloat(randomFloat()).build()) { + assertEmptyBlock(toFilter.filter().asBlock()); + } + } + public void testEmptyDoubleBlock() throws IOException { assertEmptyBlock(blockFactory.newDoubleBlockBuilder(0).build()); try (DoubleBlock toFilter = blockFactory.newDoubleBlockBuilder(0).appendNull().build()) { @@ -140,6 +155,22 @@ public void testFilterLongBlock() throws IOException { } } + public void testFilterFloatBlock() throws IOException { + try (FloatBlock toFilter = blockFactory.newFloatBlockBuilder(0).appendFloat(1).appendFloat(2).build()) { + assertFilterBlock(toFilter.filter(1)); + } + try (FloatBlock toFilter = blockFactory.newFloatBlockBuilder(1).appendFloat(randomFloat()).appendNull().build()) { + assertFilterBlock(toFilter.filter(0)); + } + try (FloatVector toFilter = blockFactory.newFloatVectorBuilder(1).appendFloat(randomFloat()).build()) { + assertFilterBlock(toFilter.filter(0).asBlock()); + + } + try (FloatVector toFilter = blockFactory.newFloatVectorBuilder(1).appendFloat(randomFloat()).appendFloat(randomFloat()).build()) { + assertFilterBlock(toFilter.filter(0).asBlock()); + } + } + public void testFilterDoubleBlock() throws IOException { try (DoubleBlock toFilter = blockFactory.newDoubleBlockBuilder(0).appendDouble(1).appendDouble(2).build()) { assertFilterBlock(toFilter.filter(1)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockTestUtils.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockTestUtils.java index b02ef6d8e9589..55e80a9124de0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockTestUtils.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockTestUtils.java @@ -17,6 +17,7 @@ import static org.elasticsearch.test.ESTestCase.between; import static org.elasticsearch.test.ESTestCase.randomBoolean; import static org.elasticsearch.test.ESTestCase.randomDouble; +import static org.elasticsearch.test.ESTestCase.randomFloat; import static org.elasticsearch.test.ESTestCase.randomInt; import static org.elasticsearch.test.ESTestCase.randomLong; import static org.elasticsearch.test.ESTestCase.randomRealisticUnicodeOfCodepointLengthBetween; @@ -31,6 +32,7 @@ public static Object randomValue(ElementType e) { return switch (e) { case INT -> randomInt(); case LONG -> randomLong(); + case FLOAT -> randomFloat(); case DOUBLE -> randomDouble(); case BYTES_REF -> new BytesRef(randomRealisticUnicodeOfCodepointLengthBetween(0, 5)); // TODO: also test spatial WKB case BOOLEAN -> randomBoolean(); @@ -90,6 +92,26 @@ public static void append(Block.Builder builder, Object value) { return; } } + if (builder instanceof FloatBlock.Builder b) { + if (value instanceof Float v) { + b.appendFloat(v); + return; + } + if (value instanceof List l) { + switch (l.size()) { + case 0 -> b.appendNull(); + case 1 -> b.appendFloat((Float) l.get(0)); + default -> { + b.beginPositionEntry(); + for (Object o : l) { + b.appendFloat((Float) o); + } + b.endPositionEntry(); + } + } + return; + } + } if (builder instanceof DoubleBlock.Builder b) { if (value instanceof Double v) { b.appendDouble(v); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockValueAsserter.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockValueAsserter.java index e03de38d637db..f9c88a504d53d 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockValueAsserter.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/BlockValueAsserter.java @@ -32,6 +32,7 @@ static void assertBlockValues(Block block, List> expectedBlockValue switch (block.elementType()) { case INT -> assertIntRowValues((IntBlock) block, firstValueIndex, valueCount, expectedRowValues); case LONG -> assertLongRowValues((LongBlock) block, firstValueIndex, valueCount, expectedRowValues); + case FLOAT -> assertFloatRowValues((FloatBlock) block, firstValueIndex, valueCount, expectedRowValues); case DOUBLE -> assertDoubleRowValues((DoubleBlock) block, firstValueIndex, valueCount, expectedRowValues); case BYTES_REF -> assertBytesRefRowValues((BytesRefBlock) block, firstValueIndex, valueCount, expectedRowValues); case BOOLEAN -> assertBooleanRowValues((BooleanBlock) block, firstValueIndex, valueCount, expectedRowValues); @@ -55,6 +56,13 @@ private static void assertLongRowValues(LongBlock block, int firstValueIndex, in } } + private static void assertFloatRowValues(FloatBlock block, int firstValueIndex, int valueCount, List expectedRowValues) { + for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) { + float expectedValue = ((Number) expectedRowValues.get(valueIndex)).floatValue(); + assertThat(block.getFloat(firstValueIndex + valueIndex), is(equalTo(expectedValue))); + } + } + private static void assertDoubleRowValues(DoubleBlock block, int firstValueIndex, int valueCount, List expectedRowValues) { for (int valueIndex = 0; valueIndex < valueCount; valueIndex++) { double expectedValue = ((Number) expectedRowValues.get(valueIndex)).doubleValue(); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FloatBlockEqualityTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FloatBlockEqualityTests.java new file mode 100644 index 0000000000000..95e9349a18fee --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/FloatBlockEqualityTests.java @@ -0,0 +1,312 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data; + +import org.elasticsearch.compute.operator.ComputeTestCase; +import org.elasticsearch.core.Releasables; + +import java.util.BitSet; +import java.util.List; + +public class FloatBlockEqualityTests extends ComputeTestCase { + + static final BlockFactory blockFactory = TestBlockFactory.getNonBreakingInstance(); + + public void testEmptyVector() { + // all these "empty" vectors should be equivalent + List vectors = List.of( + blockFactory.newFloatArrayVector(new float[] {}, 0), + blockFactory.newFloatArrayVector(new float[] { 0 }, 0), + blockFactory.newConstantFloatVector(0, 0), + blockFactory.newConstantFloatBlockWith(0, 0).filter().asVector(), + blockFactory.newFloatBlockBuilder(0).build().asVector(), + blockFactory.newFloatBlockBuilder(0).appendFloat(1).build().asVector().filter() + ); + assertAllEquals(vectors); + } + + public void testEmptyBlock() { + // all these "empty" vectors should be equivalent + List blocks = List.of( + blockFactory.newFloatArrayBlock( + new float[] {}, + 0, + new int[] { 0 }, + BitSet.valueOf(new byte[] { 0b00 }), + randomFrom(Block.MvOrdering.values()) + ), + blockFactory.newFloatArrayBlock( + new float[] { 0 }, + 0, + new int[] { 0 }, + BitSet.valueOf(new byte[] { 0b00 }), + randomFrom(Block.MvOrdering.values()) + ), + blockFactory.newConstantFloatBlockWith(0, 0), + blockFactory.newFloatBlockBuilder(0).build(), + blockFactory.newFloatBlockBuilder(0).appendFloat(1).build().filter(), + blockFactory.newFloatBlockBuilder(0).appendNull().build().filter() + ); + assertAllEquals(blocks); + Releasables.close(blocks); + } + + public void testVectorEquality() { + // all these vectors should be equivalent + List vectors = List.of( + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3 }, 3), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3 }, 3).asBlock().asVector(), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3, 4 }, 3), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3 }, 3).filter(0, 1, 2), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3, 4 }, 4).filter(0, 1, 2), + blockFactory.newFloatArrayVector(new float[] { 0, 1, 2, 3 }, 4).filter(1, 2, 3), + blockFactory.newFloatArrayVector(new float[] { 1, 4, 2, 3 }, 4).filter(0, 2, 3), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(2).appendFloat(3).build().asVector(), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(2).appendFloat(3).build().asVector().filter(0, 1, 2), + blockFactory.newFloatBlockBuilder(3) + .appendFloat(1) + .appendFloat(4) + .appendFloat(2) + .appendFloat(3) + .build() + .filter(0, 2, 3) + .asVector(), + blockFactory.newFloatBlockBuilder(3) + .appendFloat(1) + .appendFloat(4) + .appendFloat(2) + .appendFloat(3) + .build() + .asVector() + .filter(0, 2, 3) + ); + assertAllEquals(vectors); + + // all these constant-like vectors should be equivalent + List moreVectors = List.of( + blockFactory.newFloatArrayVector(new float[] { 1, 1, 1 }, 3), + blockFactory.newFloatArrayVector(new float[] { 1, 1, 1 }, 3).asBlock().asVector(), + blockFactory.newFloatArrayVector(new float[] { 1, 1, 1, 1 }, 3), + blockFactory.newFloatArrayVector(new float[] { 1, 1, 1 }, 3).filter(0, 1, 2), + blockFactory.newFloatArrayVector(new float[] { 1, 1, 1, 4 }, 4).filter(0, 1, 2), + blockFactory.newFloatArrayVector(new float[] { 3, 1, 1, 1 }, 4).filter(1, 2, 3), + blockFactory.newFloatArrayVector(new float[] { 1, 4, 1, 1 }, 4).filter(0, 2, 3), + blockFactory.newConstantFloatBlockWith(1, 3).asVector(), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(1).appendFloat(1).build().asVector(), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(1).appendFloat(1).build().asVector().filter(0, 1, 2), + blockFactory.newFloatBlockBuilder(3) + .appendFloat(1) + .appendFloat(4) + .appendFloat(1) + .appendFloat(1) + .build() + .filter(0, 2, 3) + .asVector(), + blockFactory.newFloatBlockBuilder(3) + .appendFloat(1) + .appendFloat(4) + .appendFloat(1) + .appendFloat(1) + .build() + .asVector() + .filter(0, 2, 3) + ); + assertAllEquals(moreVectors); + } + + public void testBlockEquality() { + // all these blocks should be equivalent + List blocks = List.of( + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3 }, 3).asBlock(), + new FloatArrayBlock( + new float[] { 1, 2, 3 }, + 3, + new int[] { 0, 1, 2, 3 }, + BitSet.valueOf(new byte[] { 0b000 }), + randomFrom(Block.MvOrdering.values()), + blockFactory + ), + new FloatArrayBlock( + new float[] { 1, 2, 3, 4 }, + 3, + new int[] { 0, 1, 2, 3 }, + BitSet.valueOf(new byte[] { 0b1000 }), + randomFrom(Block.MvOrdering.values()), + blockFactory + ), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3 }, 3).filter(0, 1, 2).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3, 4 }, 3).filter(0, 1, 2).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3, 4 }, 4).filter(0, 1, 2).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 4, 3 }, 4).filter(0, 1, 3).asBlock(), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(2).appendFloat(3).build(), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(2).appendFloat(3).build().filter(0, 1, 2), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(4).appendFloat(2).appendFloat(3).build().filter(0, 2, 3), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendNull().appendFloat(2).appendFloat(3).build().filter(0, 2, 3) + ); + assertAllEquals(blocks); + + // all these constant-like blocks should be equivalent + List moreBlocks = List.of( + blockFactory.newFloatArrayVector(new float[] { 9, 9 }, 2).asBlock(), + new FloatArrayBlock( + new float[] { 9, 9 }, + 2, + new int[] { 0, 1, 2 }, + BitSet.valueOf(new byte[] { 0b000 }), + randomFrom(Block.MvOrdering.values()), + blockFactory + ), + new FloatArrayBlock( + new float[] { 9, 9, 4 }, + 2, + new int[] { 0, 1, 2 }, + BitSet.valueOf(new byte[] { 0b100 }), + randomFrom(Block.MvOrdering.values()), + blockFactory + ), + blockFactory.newFloatArrayVector(new float[] { 9, 9 }, 2).filter(0, 1).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 9, 9, 4 }, 2).filter(0, 1).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 9, 9, 4 }, 3).filter(0, 1).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 9, 4, 9 }, 3).filter(0, 2).asBlock(), + blockFactory.newConstantFloatBlockWith(9, 2), + blockFactory.newFloatBlockBuilder(2).appendFloat(9).appendFloat(9).build(), + blockFactory.newFloatBlockBuilder(2).appendFloat(9).appendFloat(9).build().filter(0, 1), + blockFactory.newFloatBlockBuilder(2).appendFloat(9).appendFloat(4).appendFloat(9).build().filter(0, 2), + blockFactory.newFloatBlockBuilder(2).appendFloat(9).appendNull().appendFloat(9).build().filter(0, 2) + ); + assertAllEquals(moreBlocks); + } + + public void testVectorInequality() { + // all these vectors should NOT be equivalent + List notEqualVectors = List.of( + blockFactory.newFloatArrayVector(new float[] { 1 }, 1), + blockFactory.newFloatArrayVector(new float[] { 9 }, 1), + blockFactory.newFloatArrayVector(new float[] { 1, 2 }, 2), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3 }, 3), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 4 }, 3), + blockFactory.newConstantFloatBlockWith(9, 2).asVector(), + blockFactory.newFloatBlockBuilder(2).appendFloat(1).appendFloat(2).build().asVector().filter(1), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(2).appendFloat(5).build().asVector(), + blockFactory.newFloatBlockBuilder(1).appendFloat(1).appendFloat(2).appendFloat(3).appendFloat(4).build().asVector() + ); + assertAllNotEquals(notEqualVectors); + } + + public void testBlockInequality() { + // all these blocks should NOT be equivalent + List notEqualBlocks = List.of( + blockFactory.newFloatArrayVector(new float[] { 1 }, 1).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 9 }, 1).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 1, 2 }, 2).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 3 }, 3).asBlock(), + blockFactory.newFloatArrayVector(new float[] { 1, 2, 4 }, 3).asBlock(), + blockFactory.newConstantFloatBlockWith(9, 2), + blockFactory.newFloatBlockBuilder(2).appendFloat(1).appendFloat(2).build().filter(1), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).appendFloat(2).appendFloat(5).build(), + blockFactory.newFloatBlockBuilder(1).appendFloat(1).appendFloat(2).appendFloat(3).appendFloat(4).build(), + blockFactory.newFloatBlockBuilder(1).appendFloat(1).appendNull().build(), + blockFactory.newFloatBlockBuilder(1).appendFloat(1).appendNull().appendFloat(3).build(), + blockFactory.newFloatBlockBuilder(1).appendFloat(1).appendFloat(3).build(), + blockFactory.newFloatBlockBuilder(3).appendFloat(1).beginPositionEntry().appendFloat(2).appendFloat(3).build() + ); + assertAllNotEquals(notEqualBlocks); + } + + public void testSimpleBlockWithSingleNull() { + List blocks = List.of( + blockFactory.newFloatBlockBuilder(3).appendFloat(1.1f).appendNull().appendFloat(3.1f).build(), + blockFactory.newFloatBlockBuilder(3).appendFloat(1.1f).appendNull().appendFloat(3.1f).build() + ); + assertEquals(3, blocks.get(0).getPositionCount()); + assertTrue(blocks.get(0).isNull(1)); + assertAllEquals(blocks); + } + + public void testSimpleBlockWithManyNulls() { + int positions = randomIntBetween(1, 256); + boolean grow = randomBoolean(); + FloatBlock.Builder builder1 = blockFactory.newFloatBlockBuilder(grow ? 0 : positions); + FloatBlock.Builder builder2 = blockFactory.newFloatBlockBuilder(grow ? 0 : positions); + for (int p = 0; p < positions; p++) { + builder1.appendNull(); + builder2.appendNull(); + } + FloatBlock block1 = builder1.build(); + FloatBlock block2 = builder2.build(); + assertEquals(positions, block1.getPositionCount()); + assertTrue(block1.mayHaveNulls()); + assertTrue(block1.isNull(0)); + + List blocks = List.of(block1, block2); + assertAllEquals(blocks); + } + + public void testSimpleBlockWithSingleMultiValue() { + List blocks = List.of( + blockFactory.newFloatBlockBuilder(1).beginPositionEntry().appendFloat(1.1f).appendFloat(2.2f).build(), + blockFactory.newFloatBlockBuilder(1).beginPositionEntry().appendFloat(1.1f).appendFloat(2.2f).build() + ); + assert blocks.get(0).getPositionCount() == 1 && blocks.get(0).getValueCount(0) == 2; + assertAllEquals(blocks); + } + + public void testSimpleBlockWithManyMultiValues() { + int positions = randomIntBetween(1, 256); + boolean grow = randomBoolean(); + FloatBlock.Builder builder1 = blockFactory.newFloatBlockBuilder(grow ? 0 : positions); + FloatBlock.Builder builder2 = blockFactory.newFloatBlockBuilder(grow ? 0 : positions); + FloatBlock.Builder builder3 = blockFactory.newFloatBlockBuilder(grow ? 0 : positions); + for (int pos = 0; pos < positions; pos++) { + builder1.beginPositionEntry(); + builder2.beginPositionEntry(); + builder3.beginPositionEntry(); + int values = randomIntBetween(1, 16); + for (int i = 0; i < values; i++) { + float value = randomFloat(); + builder1.appendFloat(value); + builder2.appendFloat(value); + builder3.appendFloat(value); + } + builder1.endPositionEntry(); + builder2.endPositionEntry(); + builder3.endPositionEntry(); + } + FloatBlock block1 = builder1.build(); + FloatBlock block2 = builder2.build(); + FloatBlock block3 = builder3.build(); + + assertEquals(positions, block1.getPositionCount()); + assertAllEquals(List.of(block1, block2, block3)); + } + + static void assertAllEquals(List objs) { + for (Object obj1 : objs) { + for (Object obj2 : objs) { + assertEquals(obj1, obj2); + assertEquals(obj2, obj1); + // equal objects must generate the same hash code + assertEquals(obj1.hashCode(), obj2.hashCode()); + } + } + } + + static void assertAllNotEquals(List objs) { + for (Object obj1 : objs) { + for (Object obj2 : objs) { + if (obj1 == obj2) { + continue; // skip self + } + assertNotEquals(obj1, obj2); + assertNotEquals(obj2, obj1); + // unequal objects SHOULD generate the different hash code + assertNotEquals(obj1.hashCode(), obj2.hashCode()); + } + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/VectorBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/VectorBuilderTests.java index d41ccc26bfe49..3ab02ac5488bc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/VectorBuilderTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/VectorBuilderTests.java @@ -116,6 +116,7 @@ private Vector.Builder vectorBuilder(int estimatedSize, BlockFactory blockFactor case NULL, DOC, COMPOSITE, UNKNOWN -> throw new UnsupportedOperationException(); case BOOLEAN -> blockFactory.newBooleanVectorBuilder(estimatedSize); case BYTES_REF -> blockFactory.newBytesRefVectorBuilder(estimatedSize); + case FLOAT -> blockFactory.newFloatVectorBuilder(estimatedSize); case DOUBLE -> blockFactory.newDoubleVectorBuilder(estimatedSize); case INT -> blockFactory.newIntVectorBuilder(estimatedSize); case LONG -> blockFactory.newLongVectorBuilder(estimatedSize); @@ -135,6 +136,11 @@ private void fill(Vector.Builder builder, Vector from) { ((BytesRefVector.Builder) builder).appendBytesRef(((BytesRefVector) from).getBytesRef(p, new BytesRef())); } } + case FLOAT -> { + for (int p = 0; p < from.getPositionCount(); p++) { + ((FloatVector.Builder) builder).appendFloat(((FloatVector) from).getFloat(p)); + } + } case DOUBLE -> { for (int p = 0; p < from.getPositionCount(); p++) { ((DoubleVector.Builder) builder).appendDouble(((DoubleVector) from).getDouble(p)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/VectorFixedBuilderTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/VectorFixedBuilderTests.java index f6b1acf4131d2..1086280af9df0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/VectorFixedBuilderTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/VectorFixedBuilderTests.java @@ -119,6 +119,7 @@ private Vector.Builder vectorBuilder(int size, BlockFactory blockFactory) { case NULL, BYTES_REF, DOC, COMPOSITE, UNKNOWN -> throw new UnsupportedOperationException(); case BOOLEAN -> blockFactory.newBooleanVectorFixedBuilder(size); case DOUBLE -> blockFactory.newDoubleVectorFixedBuilder(size); + case FLOAT -> blockFactory.newFloatVectorFixedBuilder(size); case INT -> blockFactory.newIntVectorFixedBuilder(size); case LONG -> blockFactory.newLongVectorFixedBuilder(size); }; @@ -132,6 +133,11 @@ private void fill(Vector.Builder builder, Vector from) { ((BooleanVector.FixedBuilder) builder).appendBoolean(((BooleanVector) from).getBoolean(p)); } } + case FLOAT -> { + for (int p = 0; p < from.getPositionCount(); p++) { + ((FloatVector.Builder) builder).appendFloat(((FloatVector) from).getFloat(p)); + } + } case DOUBLE -> { for (int p = 0; p < from.getPositionCount(); p++) { ((DoubleVector.FixedBuilder) builder).appendDouble(((DoubleVector) from).getDouble(p)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java new file mode 100644 index 0000000000000..9e1bc145ad4ca --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java @@ -0,0 +1,368 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BitArray; +import org.elasticsearch.common.util.MockBigArrays; +import org.elasticsearch.common.util.MockPageCacheRecycler; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.TestBlockFactory; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.equalTo; + +public abstract class BucketedSortTestCase extends ESTestCase { + /** + * Build a {@link T} to test. Sorts built by this method shouldn't need scores. + */ + protected abstract T build(SortOrder sortOrder, int bucketSize); + + /** + * Build the expected correctly typed value for a value. + */ + protected abstract Object expectedValue(double v); + + /** + * A random value for testing, with the appropriate precision for the type we're testing. + */ + protected abstract double randomValue(); + + /** + * Collect a value into the sort. + * @param value value to collect, always sent as double just to have + * a number to test. Subclasses should cast to their favorite types + */ + protected abstract void collect(T sort, double value, int bucket); + + protected abstract void merge(T sort, int groupId, T other, int otherGroupId); + + protected abstract Block toBlock(T sort, BlockFactory blockFactory, IntVector selected); + + protected abstract void assertBlockTypeAndValues(Block block, Object... values); + + public final void testNeverCalled() { + SortOrder order = randomFrom(SortOrder.values()); + try (T sort = build(order, 1)) { + assertBlock(sort, randomNonNegativeInt()); + } + } + + public final void testSingleDoc() { + try (T sort = build(randomFrom(SortOrder.values()), 1)) { + collect(sort, 1, 0); + + assertBlock(sort, 0, expectedValue(1)); + } + } + + public final void testNonCompetitive() { + try (T sort = build(SortOrder.DESC, 1)) { + collect(sort, 2, 0); + collect(sort, 1, 0); + + assertBlock(sort, 0, expectedValue(2)); + } + } + + public final void testCompetitive() { + try (T sort = build(SortOrder.DESC, 1)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + + assertBlock(sort, 0, expectedValue(2)); + } + } + + public final void testNegativeValue() { + try (T sort = build(SortOrder.DESC, 1)) { + collect(sort, -1, 0); + assertBlock(sort, 0, expectedValue(-1)); + } + } + + public final void testSomeBuckets() { + try (T sort = build(SortOrder.DESC, 1)) { + collect(sort, 2, 0); + collect(sort, 2, 1); + collect(sort, 2, 2); + collect(sort, 3, 0); + + assertBlock(sort, 0, expectedValue(3)); + assertBlock(sort, 1, expectedValue(2)); + assertBlock(sort, 2, expectedValue(2)); + assertBlock(sort, 3); + } + } + + public final void testBucketGaps() { + try (T sort = build(SortOrder.DESC, 1)) { + collect(sort, 2, 0); + collect(sort, 2, 2); + + assertBlock(sort, 0, expectedValue(2)); + assertBlock(sort, 1); + assertBlock(sort, 2, expectedValue(2)); + assertBlock(sort, 3); + } + } + + public final void testBucketsOutOfOrder() { + try (T sort = build(SortOrder.DESC, 1)) { + collect(sort, 2, 1); + collect(sort, 2, 0); + + assertBlock(sort, 0, expectedValue(2.0)); + assertBlock(sort, 1, expectedValue(2.0)); + assertBlock(sort, 2); + } + } + + public final void testManyBuckets() { + // Collect the buckets in random order + int[] buckets = new int[10000]; + for (int b = 0; b < buckets.length; b++) { + buckets[b] = b; + } + Collections.shuffle(Arrays.asList(buckets), random()); + + double[] maxes = new double[buckets.length]; + + try (T sort = build(SortOrder.DESC, 1)) { + for (int b : buckets) { + maxes[b] = 2; + collect(sort, 2, b); + if (randomBoolean()) { + maxes[b] = 3; + collect(sort, 3, b); + } + if (randomBoolean()) { + collect(sort, -1, b); + } + } + for (int b = 0; b < buckets.length; b++) { + assertBlock(sort, b, expectedValue(maxes[b])); + } + assertBlock(sort, buckets.length); + } + } + + public final void testTwoHitsDesc() { + try (T sort = build(SortOrder.DESC, 2)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + collect(sort, 3, 0); + + assertBlock(sort, 0, expectedValue(3), expectedValue(2)); + } + } + + public final void testTwoHitsAsc() { + try (T sort = build(SortOrder.ASC, 2)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + collect(sort, 3, 0); + + assertBlock(sort, 0, expectedValue(1), expectedValue(2)); + } + } + + public final void testTwoHitsTwoBucket() { + try (T sort = build(SortOrder.DESC, 2)) { + collect(sort, 1, 0); + collect(sort, 1, 1); + collect(sort, 2, 0); + collect(sort, 2, 1); + collect(sort, 3, 0); + collect(sort, 3, 1); + collect(sort, 4, 1); + + assertBlock(sort, 0, expectedValue(3), expectedValue(2)); + assertBlock(sort, 1, expectedValue(4), expectedValue(3)); + } + } + + public final void testManyBucketsManyHits() { + // Set the values in random order + double[] values = new double[10000]; + for (int v = 0; v < values.length; v++) { + values[v] = randomValue(); + } + Collections.shuffle(Arrays.asList(values), random()); + + int buckets = between(2, 100); + int bucketSize = between(2, 100); + try (T sort = build(SortOrder.DESC, bucketSize)) { + BitArray[] bucketUsed = new BitArray[buckets]; + Arrays.setAll(bucketUsed, i -> new BitArray(values.length, bigArrays())); + for (int doc = 0; doc < values.length; doc++) { + for (int bucket = 0; bucket < buckets; bucket++) { + if (randomBoolean()) { + bucketUsed[bucket].set(doc); + collect(sort, values[doc], bucket); + } + } + } + for (int bucket = 0; bucket < buckets; bucket++) { + List bucketValues = new ArrayList<>(values.length); + for (int doc = 0; doc < values.length; doc++) { + if (bucketUsed[bucket].get(doc)) { + bucketValues.add(values[doc]); + } + } + bucketUsed[bucket].close(); + assertBlock( + sort, + bucket, + bucketValues.stream().sorted((lhs, rhs) -> rhs.compareTo(lhs)).limit(bucketSize).map(this::expectedValue).toArray() + ); + } + assertBlock(sort, buckets); + } + } + + public final void testMergeHeapToHeap() { + try (T sort = build(SortOrder.ASC, 3)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + collect(sort, 3, 0); + + try (T other = build(SortOrder.ASC, 3)) { + collect(other, 1, 0); + collect(other, 2, 0); + collect(other, 3, 0); + + merge(sort, 0, other, 0); + } + + assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2)); + } + } + + public final void testMergeNoHeapToNoHeap() { + try (T sort = build(SortOrder.ASC, 3)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + + try (T other = build(SortOrder.ASC, 3)) { + collect(other, 1, 0); + collect(other, 2, 0); + + merge(sort, 0, other, 0); + } + + assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2)); + } + } + + public final void testMergeHeapToNoHeap() { + try (T sort = build(SortOrder.ASC, 3)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + + try (T other = build(SortOrder.ASC, 3)) { + collect(other, 1, 0); + collect(other, 2, 0); + collect(other, 3, 0); + + merge(sort, 0, other, 0); + } + + assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2)); + } + } + + public final void testMergeNoHeapToHeap() { + try (T sort = build(SortOrder.ASC, 3)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + collect(sort, 3, 0); + + try (T other = build(SortOrder.ASC, 3)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + + merge(sort, 0, other, 0); + } + + assertBlock(sort, 0, expectedValue(1), expectedValue(1), expectedValue(2)); + } + } + + public final void testMergeHeapToEmpty() { + try (T sort = build(SortOrder.ASC, 3)) { + try (T other = build(SortOrder.ASC, 3)) { + collect(other, 1, 0); + collect(other, 2, 0); + collect(other, 3, 0); + + merge(sort, 0, other, 0); + } + + assertBlock(sort, 0, expectedValue(1), expectedValue(2), expectedValue(3)); + } + } + + public final void testMergeEmptyToHeap() { + try (T sort = build(SortOrder.ASC, 3)) { + collect(sort, 1, 0); + collect(sort, 2, 0); + collect(sort, 3, 0); + + try (T other = build(SortOrder.ASC, 3)) { + merge(sort, 0, other, 0); + } + + assertBlock(sort, 0, expectedValue(1), expectedValue(2), expectedValue(3)); + } + } + + public final void testMergeEmptyToEmpty() { + try (T sort = build(SortOrder.ASC, 3)) { + try (T other = build(SortOrder.ASC, 3)) { + merge(sort, 0, other, randomNonNegativeInt()); + } + + assertBlock(sort, 0); + } + } + + private void assertBlock(T sort, int groupId, Object... values) { + var blockFactory = TestBlockFactory.getNonBreakingInstance(); + + try (var intVector = blockFactory.newConstantIntVector(groupId, 1)) { + var block = toBlock(sort, blockFactory, intVector); + + assertThat(block.getPositionCount(), equalTo(1)); + assertThat(block.getTotalValueCount(), equalTo(values.length)); + + if (values.length == 0) { + assertThat(block.elementType(), equalTo(ElementType.NULL)); + assertThat(block.isNull(0), equalTo(true)); + } else { + assertBlockTypeAndValues(block, values); + } + } + } + + protected final BigArrays bigArrays() { + return new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService()); + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java new file mode 100644 index 0000000000000..43b5caa092b9a --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/DoubleBucketedSortTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.search.sort.SortOrder; + +import static org.hamcrest.Matchers.equalTo; + +public class DoubleBucketedSortTests extends BucketedSortTestCase { + @Override + protected DoubleBucketedSort build(SortOrder sortOrder, int bucketSize) { + return new DoubleBucketedSort(bigArrays(), sortOrder, bucketSize); + } + + @Override + protected Object expectedValue(double v) { + return v; + } + + @Override + protected double randomValue() { + return randomDoubleBetween(Double.MIN_VALUE, Double.MAX_VALUE, true); + } + + @Override + protected void collect(DoubleBucketedSort sort, double value, int bucket) { + sort.collect(value, bucket); + } + + @Override + protected void merge(DoubleBucketedSort sort, int groupId, DoubleBucketedSort other, int otherGroupId) { + sort.merge(groupId, other, otherGroupId); + } + + @Override + protected Block toBlock(DoubleBucketedSort sort, BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + @Override + protected void assertBlockTypeAndValues(Block block, Object... values) { + assertThat(block.elementType(), equalTo(ElementType.DOUBLE)); + var typedBlock = (DoubleBlock) block; + for (int i = 0; i < values.length; i++) { + assertThat(typedBlock.getDouble(i), equalTo(values[i])); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java new file mode 100644 index 0000000000000..8b3d288339037 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/FloatBucketedSortTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.search.sort.SortOrder; + +import static org.hamcrest.Matchers.equalTo; + +public class FloatBucketedSortTests extends BucketedSortTestCase { + @Override + protected FloatBucketedSort build(SortOrder sortOrder, int bucketSize) { + return new FloatBucketedSort(bigArrays(), sortOrder, bucketSize); + } + + @Override + protected Object expectedValue(double v) { + return v; + } + + @Override + protected double randomValue() { + return randomFloatBetween(Float.MIN_VALUE, Float.MAX_VALUE, true); + } + + @Override + protected void collect(FloatBucketedSort sort, double value, int bucket) { + sort.collect((float) value, bucket); + } + + @Override + protected void merge(FloatBucketedSort sort, int groupId, FloatBucketedSort other, int otherGroupId) { + sort.merge(groupId, other, otherGroupId); + } + + @Override + protected Block toBlock(FloatBucketedSort sort, BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + @Override + protected void assertBlockTypeAndValues(Block block, Object... values) { + assertThat(block.elementType(), equalTo(ElementType.FLOAT)); + var typedBlock = (FloatBlock) block; + for (int i = 0; i < values.length; i++) { + assertThat((double) typedBlock.getFloat(i), equalTo(values[i])); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java new file mode 100644 index 0000000000000..70d0a79ea7473 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/IntBucketedSortTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.search.sort.SortOrder; + +import static org.hamcrest.Matchers.equalTo; + +public class IntBucketedSortTests extends BucketedSortTestCase { + @Override + protected IntBucketedSort build(SortOrder sortOrder, int bucketSize) { + return new IntBucketedSort(bigArrays(), sortOrder, bucketSize); + } + + @Override + protected Object expectedValue(double v) { + return (int) v; + } + + @Override + protected double randomValue() { + return randomInt(); + } + + @Override + protected void collect(IntBucketedSort sort, double value, int bucket) { + sort.collect((int) value, bucket); + } + + @Override + protected void merge(IntBucketedSort sort, int groupId, IntBucketedSort other, int otherGroupId) { + sort.merge(groupId, other, otherGroupId); + } + + @Override + protected Block toBlock(IntBucketedSort sort, BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + @Override + protected void assertBlockTypeAndValues(Block block, Object... values) { + assertThat(block.elementType(), equalTo(ElementType.INT)); + var typedBlock = (IntBlock) block; + for (int i = 0; i < values.length; i++) { + assertThat(typedBlock.getInt(i), equalTo(values[i])); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java new file mode 100644 index 0000000000000..bceed3b1d95b5 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/LongBucketedSortTests.java @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.data.sort; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.search.sort.SortOrder; + +import static org.hamcrest.Matchers.equalTo; + +public class LongBucketedSortTests extends BucketedSortTestCase { + @Override + protected LongBucketedSort build(SortOrder sortOrder, int bucketSize) { + return new LongBucketedSort(bigArrays(), sortOrder, bucketSize); + } + + @Override + protected Object expectedValue(double v) { + return (long) v; + } + + @Override + protected double randomValue() { + // 2L^50 fits in the mantisa of a double which the test sort of needs. + return randomLongBetween(-2L ^ 50, 2L ^ 50); + } + + @Override + protected void collect(LongBucketedSort sort, double value, int bucket) { + sort.collect((long) value, bucket); + } + + @Override + protected void merge(LongBucketedSort sort, int groupId, LongBucketedSort other, int otherGroupId) { + sort.merge(groupId, other, otherGroupId); + } + + @Override + protected Block toBlock(LongBucketedSort sort, BlockFactory blockFactory, IntVector selected) { + return sort.toBlock(blockFactory, selected); + } + + @Override + protected void assertBlockTypeAndValues(Block block, Object... values) { + assertThat(block.elementType(), equalTo(ElementType.LONG)); + var typedBlock = (LongBlock) block; + for (int i = 0; i < values.length; i++) { + assertThat(typedBlock.getLong(i), equalTo(values[i])); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java new file mode 100644 index 0000000000000..66bcf2a57e393 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java @@ -0,0 +1,2020 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.lucene; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleDocValuesField; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Iterators; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BooleanBlock; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.BytesRefVector; +import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.DoubleVector; +import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.IntBlock; +import org.elasticsearch.compute.data.IntVector; +import org.elasticsearch.compute.data.LongBlock; +import org.elasticsearch.compute.data.LongVector; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.compute.data.TestBlockFactory; +import org.elasticsearch.compute.operator.AnyOperatorTestCase; +import org.elasticsearch.compute.operator.CannedSourceOperator; +import org.elasticsearch.compute.operator.Driver; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.DriverRunner; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.compute.operator.Operator; +import org.elasticsearch.compute.operator.PageConsumerOperator; +import org.elasticsearch.compute.operator.SequenceLongBlockSourceOperator; +import org.elasticsearch.compute.operator.SourceOperator; +import org.elasticsearch.compute.operator.TestResultPageSinkOperator; +import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.SourceLoader; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.fetch.StoredFieldsSpec; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.StringUtils; +import org.hamcrest.Matcher; +import org.junit.After; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.core.type.DataType.IP; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.nullValue; +import static org.hamcrest.Matchers.oneOf; +import static org.hamcrest.Matchers.sameInstance; + +/** + * These tests are partial duplicates of the tests in ValuesSourceReaderOperatorTests, and focus on testing the behaviour + * of the ValuesSourceReaderOperator, but with a few key differences: + *
      + *
    • Multiple indexes and index mappings are defined and tested
      • + *
    • + * Most primitive types also include a field with prefix 'str_' which is stored and mapped as a string, + * but expected to be extracted and converted directly to the primitive type. + * For example: "str_long": "1" should be read directly into a field named "str_long" of type "long" and value 1. + * This tests the ability of the BlockLoader.convert(Block) method to convert a string to a primitive type. + *
      • + *
    • + * Each index has a few additional custom fields that are stored as specific types, but should be converted to strings by the + * BlockLoader.convert(Block) method. These fields are: + *
        + *
      • ip: stored as an IP type, but should be converted to a string
        • + *
      • duration: stored as a long type, but should be converted to a string
        • + *
      + * One index stores them as IP and long types, and the other as keyword types, so we test the behaviour of the + * 'union types' capabilities of the ValuesSourceReaderOperator class. + *
      • + *
    + * Since this test does not have access to the type conversion code in the ESQL module, we have mocks for that behaviour + * in the inner classes TestTypeConvertingBlockLoader and TestBlockConverter. + */ +@SuppressWarnings("resource") +public class ValueSourceReaderTypeConversionTests extends AnyOperatorTestCase { + private static final String[] PREFIX = new String[] { "a", "b", "c" }; + private static final Map INDICES = new LinkedHashMap<>(); + static { + addIndex( + Map.of( + "ip", + new TestFieldType<>("ip", IP, d -> "192.169.0." + d % 256, Checks::unionIPsAsStrings), + "duration", + new TestFieldType<>("duration", DataType.LONG, d -> (long) d, Checks::unionDurationsAsStrings) + ) + ); + addIndex( + Map.of( + "ip", + new TestFieldType<>("ip", DataType.KEYWORD, d -> "192.169.0." + d % 256, Checks::unionIPsAsStrings), + "duration", + new TestFieldType<>("duration", DataType.KEYWORD, d -> Integer.toString(d), Checks::unionDurationsAsStrings) + ) + ); + } + + static void addIndex(Map> fieldTypes) { + String indexKey = "index" + (INDICES.size() + 1); + INDICES.put(indexKey, new TestIndexMappingConfig(indexKey, INDICES.size(), fieldTypes)); + } + + private record TestIndexMappingConfig(String indexName, int shardIdx, Map> fieldTypes) {} + + private record TestFieldType(String name, DataType dataType, Function valueGenerator, CheckResults checkResults) {} + + private final Map directories = new HashMap<>(); + private final Map mapperServices = new HashMap<>(); + private final Map readers = new HashMap<>(); + private static final Map> keyToTags = new HashMap<>(); + + @After + public void closeIndex() throws IOException { + IOUtils.close(readers.values()); + IOUtils.close(directories.values()); + } + + private Directory directory(String indexKey) { + return directories.computeIfAbsent(indexKey, k -> newDirectory()); + } + + private MapperService mapperService(String indexKey) { + return mapperServices.get(indexKey); + } + + private List initShardContexts() { + return INDICES.keySet() + .stream() + .map(index -> new ValuesSourceReaderOperator.ShardContext(reader(index), () -> SourceLoader.FROM_STORED_SOURCE)) + .toList(); + } + + private IndexReader reader(String indexKey) { + if (readers.get(indexKey) == null) { + try { + initIndex(indexKey, 100, 10); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return readers.get(indexKey); + } + + @Override + protected Operator.OperatorFactory simple() { + return factory(initShardContexts(), mapperService("index1").fieldType("long"), ElementType.LONG); + } + + public static Operator.OperatorFactory factory( + List shardContexts, + MappedFieldType ft, + ElementType elementType + ) { + return factory(shardContexts, ft.name(), elementType, ft.blockLoader(null)); + } + + private static Operator.OperatorFactory factory( + List shardContexts, + String name, + ElementType elementType, + BlockLoader loader + ) { + return new ValuesSourceReaderOperator.Factory(List.of(new ValuesSourceReaderOperator.FieldInfo(name, elementType, shardIdx -> { + if (shardIdx < 0 || shardIdx >= INDICES.size()) { + fail("unexpected shardIdx [" + shardIdx + "]"); + } + return loader; + })), shardContexts, 0); + } + + protected SourceOperator simpleInput(DriverContext context, int size) { + return simpleInput(context, size, commitEvery(size), randomPageSize()); + } + + private int commitEvery(int numDocs) { + return Math.max(1, (int) Math.ceil((double) numDocs / 10)); + } + + private SourceOperator simpleInput(DriverContext context, int size, int commitEvery, int pageSize) { + List shardContexts = new ArrayList<>(); + try { + for (String indexKey : INDICES.keySet()) { + initIndex(indexKey, size, commitEvery); + shardContexts.add(new LuceneSourceOperatorTests.MockShardContext(reader(indexKey), INDICES.get(indexKey).shardIdx)); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + var luceneFactory = new LuceneSourceOperator.Factory( + shardContexts, + ctx -> new MatchAllDocsQuery(), + DataPartitioning.SHARD, + 1,// randomIntBetween(1, 10), + pageSize, + LuceneOperator.NO_LIMIT + ); + return luceneFactory.get(context); + } + + private void initMapping(String indexKey) throws IOException { + TestIndexMappingConfig indexMappingConfig = INDICES.get(indexKey); + mapperServices.put(indexKey, new MapperServiceTestCase() { + }.createMapperService(MapperServiceTestCase.mapping(b -> { + fieldExamples(b, "key", "integer"); // unique key per-index to use for looking up test values to compare to + fieldExamples(b, "indexKey", "keyword"); // index name (can be used to choose index-specific test values) + fieldExamples(b, "int", "integer"); + fieldExamples(b, "short", "short"); + fieldExamples(b, "byte", "byte"); + fieldExamples(b, "long", "long"); + fieldExamples(b, "double", "double"); + fieldExamples(b, "kwd", "keyword"); + b.startObject("stored_kwd").field("type", "keyword").field("store", true).endObject(); + b.startObject("mv_stored_kwd").field("type", "keyword").field("store", true).endObject(); + + simpleField(b, "missing_text", "text"); + + for (Map.Entry> entry : indexMappingConfig.fieldTypes.entrySet()) { + String fieldName = entry.getKey(); + TestFieldType fieldType = entry.getValue(); + simpleField(b, fieldName, fieldType.dataType.typeName()); + } + }))); + } + + private void initIndex(String indexKey, int size, int commitEvery) throws IOException { + initMapping(indexKey); + readers.put(indexKey, initIndex(indexKey, directory(indexKey), size, commitEvery)); + } + + private IndexReader initIndex(String indexKey, Directory directory, int size, int commitEvery) throws IOException { + keyToTags.computeIfAbsent(indexKey, k -> new HashMap<>()).clear(); + TestIndexMappingConfig indexMappingConfig = INDICES.get(indexKey); + try ( + IndexWriter writer = new IndexWriter( + directory, + newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH) + ) + ) { + for (int d = 0; d < size; d++) { + XContentBuilder source = JsonXContent.contentBuilder(); + source.startObject(); + source.field("key", d); // documents in this index have a unique key, from which most other values can be derived + source.field("indexKey", indexKey); // all documents in this index have the same indexKey + + source.field("long", d); + source.field("str_long", Long.toString(d)); + source.startArray("mv_long"); + for (int v = 0; v <= d % 3; v++) { + source.value(-1_000L * d + v); + } + source.endArray(); + source.field("source_long", (long) d); + source.startArray("mv_source_long"); + for (int v = 0; v <= d % 3; v++) { + source.value(-1_000L * d + v); + } + source.endArray(); + + source.field("int", d); + source.field("str_int", Integer.toString(d)); + source.startArray("mv_int"); + for (int v = 0; v <= d % 3; v++) { + source.value(1_000 * d + v); + } + source.endArray(); + source.field("source_int", d); + source.startArray("mv_source_int"); + for (int v = 0; v <= d % 3; v++) { + source.value(1_000 * d + v); + } + source.endArray(); + + source.field("short", (short) d); + source.field("str_short", Short.toString((short) d)); + source.startArray("mv_short"); + for (int v = 0; v <= d % 3; v++) { + source.value((short) (2_000 * d + v)); + } + source.endArray(); + source.field("source_short", (short) d); + source.startArray("mv_source_short"); + for (int v = 0; v <= d % 3; v++) { + source.value((short) (2_000 * d + v)); + } + source.endArray(); + + source.field("byte", (byte) d); + source.field("str_byte", Byte.toString((byte) d)); + source.startArray("mv_byte"); + for (int v = 0; v <= d % 3; v++) { + source.value((byte) (3_000 * d + v)); + } + source.endArray(); + source.field("source_byte", (byte) d); + source.startArray("mv_source_byte"); + for (int v = 0; v <= d % 3; v++) { + source.value((byte) (3_000 * d + v)); + } + source.endArray(); + + source.field("double", d / 123_456d); + source.field("str_double", Double.toString(d / 123_456d)); + source.startArray("mv_double"); + for (int v = 0; v <= d % 3; v++) { + source.value(d / 123_456d + v); + } + source.endArray(); + source.field("source_double", d / 123_456d); + source.startArray("mv_source_double"); + for (int v = 0; v <= d % 3; v++) { + source.value(d / 123_456d + v); + } + source.endArray(); + + String tag = keyToTags.get(indexKey).computeIfAbsent(d, k -> "tag-" + randomIntBetween(1, 5)); + source.field("kwd", tag); + source.field("str_kwd", tag); + source.startArray("mv_kwd"); + for (int v = 0; v <= d % 3; v++) { + source.value(PREFIX[v] + d); + } + source.endArray(); + source.field("stored_kwd", Integer.toString(d)); + source.startArray("mv_stored_kwd"); + for (int v = 0; v <= d % 3; v++) { + source.value(PREFIX[v] + d); + } + source.endArray(); + source.field("source_kwd", Integer.toString(d)); + source.startArray("mv_source_kwd"); + for (int v = 0; v <= d % 3; v++) { + source.value(PREFIX[v] + d); + } + source.endArray(); + + source.field("text", Integer.toString(d)); + source.startArray("mv_text"); + for (int v = 0; v <= d % 3; v++) { + source.value(PREFIX[v] + d); + } + source.endArray(); + + for (Map.Entry> entry : indexMappingConfig.fieldTypes.entrySet()) { + String fieldName = entry.getKey(); + TestFieldType fieldType = entry.getValue(); + source.field(fieldName, fieldType.valueGenerator.apply(d)); + } + + source.endObject(); + + ParsedDocument doc = mapperService(indexKey).documentParser() + .parseDocument( + new SourceToParse("id" + d, BytesReference.bytes(source), XContentType.JSON), + mapperService(indexKey).mappingLookup() + ); + writer.addDocuments(doc.docs()); + + if (d % commitEvery == commitEvery - 1) { + writer.commit(); + } + } + } + return DirectoryReader.open(directory); + } + + @Override + protected Matcher expectedDescriptionOfSimple() { + return equalTo("ValuesSourceReaderOperator[fields = [long]]"); + } + + @Override + protected Matcher expectedToStringOfSimple() { + return expectedDescriptionOfSimple(); + } + + public void testLoadAll() { + DriverContext driverContext = driverContext(); + loadSimpleAndAssert( + driverContext, + CannedSourceOperator.collectPages(simpleInput(driverContext, between(100, 5000))), + Block.MvOrdering.SORTED_ASCENDING, + Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING + ); + } + + public void testLoadAllInOnePage() { + DriverContext driverContext = driverContext(); + loadSimpleAndAssert( + driverContext, + List.of(CannedSourceOperator.mergePages(CannedSourceOperator.collectPages(simpleInput(driverContext, between(100, 5000))))), + Block.MvOrdering.UNORDERED, + Block.MvOrdering.UNORDERED + ); + } + + public void testManySingleDocPages() { + String indexKey = "index1"; + DriverContext driverContext = driverContext(); + int numDocs = between(10, 100); + List input = CannedSourceOperator.collectPages(simpleInput(driverContext, numDocs, between(1, numDocs), 1)); + Randomness.shuffle(input); + List shardContexts = initShardContexts(); + List operators = new ArrayList<>(); + Checks checks = new Checks(Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING); + FieldCase testCase = new FieldCase( + new KeywordFieldMapper.KeywordFieldType("kwd"), + ElementType.BYTES_REF, + checks::tags, + StatusChecks::keywordsFromDocValues + ); + // TODO: Add index2 + operators.add( + new ValuesSourceReaderOperator.Factory( + List.of(testCase.info, fieldInfo(mapperService(indexKey).fieldType("key"), ElementType.INT)), + shardContexts, + 0 + ).get(driverContext) + ); + List results = drive(operators, input.iterator(), driverContext); + assertThat(results, hasSize(input.size())); + for (Page page : results) { + assertThat(page.getBlockCount(), equalTo(3)); + IntVector keys = page.getBlock(2).asVector(); + for (int p = 0; p < page.getPositionCount(); p++) { + int key = keys.getInt(p); + testCase.checkResults.check(page.getBlock(1), p, key, indexKey); + } + } + } + + public void testEmpty() { + DriverContext driverContext = driverContext(); + loadSimpleAndAssert( + driverContext, + CannedSourceOperator.collectPages(simpleInput(driverContext, 0)), + Block.MvOrdering.UNORDERED, + Block.MvOrdering.UNORDERED + ); + } + + public void testLoadAllInOnePageShuffled() { + DriverContext driverContext = driverContext(); + Page source = CannedSourceOperator.mergePages(CannedSourceOperator.collectPages(simpleInput(driverContext, between(100, 5000)))); + List shuffleList = new ArrayList<>(); + IntStream.range(0, source.getPositionCount()).forEach(shuffleList::add); + Randomness.shuffle(shuffleList); + int[] shuffleArray = shuffleList.stream().mapToInt(Integer::intValue).toArray(); + Block[] shuffledBlocks = new Block[source.getBlockCount()]; + for (int b = 0; b < shuffledBlocks.length; b++) { + shuffledBlocks[b] = source.getBlock(b).filter(shuffleArray); + } + source = new Page(shuffledBlocks); + loadSimpleAndAssert(driverContext, List.of(source), Block.MvOrdering.UNORDERED, Block.MvOrdering.UNORDERED); + } + + private static ValuesSourceReaderOperator.FieldInfo fieldInfo(MappedFieldType ft, ElementType elementType) { + return new ValuesSourceReaderOperator.FieldInfo(ft.name(), elementType, shardIdx -> getBlockLoaderFor(shardIdx, ft, null)); + } + + private static ValuesSourceReaderOperator.FieldInfo fieldInfo(MappedFieldType ft, MappedFieldType ftX, ElementType elementType) { + return new ValuesSourceReaderOperator.FieldInfo(ft.name(), elementType, shardIdx -> getBlockLoaderFor(shardIdx, ft, ftX)); + } + + private ValuesSourceReaderOperator.FieldInfo fieldInfo(String fieldName, ElementType elementType, DataType toType) { + return new ValuesSourceReaderOperator.FieldInfo(fieldName, elementType, shardIdx -> getBlockLoaderFor(shardIdx, fieldName, toType)); + } + + private static MappedFieldType.BlockLoaderContext blContext() { + return new MappedFieldType.BlockLoaderContext() { + @Override + public String indexName() { + return "test_index"; + } + + @Override + public MappedFieldType.FieldExtractPreference fieldExtractPreference() { + return MappedFieldType.FieldExtractPreference.NONE; + } + + @Override + public SearchLookup lookup() { + throw new UnsupportedOperationException(); + } + + @Override + public Set sourcePaths(String name) { + return Set.of(name); + } + + @Override + public String parentField(String field) { + return null; + } + + @Override + public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { + return FieldNamesFieldMapper.FieldNamesFieldType.get(true); + } + }; + } + + private void loadSimpleAndAssert( + DriverContext driverContext, + List input, + Block.MvOrdering booleanAndNumericalDocValuesMvOrdering, + Block.MvOrdering bytesRefDocValuesMvOrdering + ) { + List cases = infoAndChecksForEachType(booleanAndNumericalDocValuesMvOrdering, bytesRefDocValuesMvOrdering); + List shardContexts = initShardContexts(); + List operators = new ArrayList<>(); + operators.add( + new ValuesSourceReaderOperator.Factory( + List.of( + fieldInfo(mapperService("index1").fieldType("key"), ElementType.INT), + fieldInfo(mapperService("index1").fieldType("indexKey"), ElementType.BYTES_REF) + ), + shardContexts, + 0 + ).get(driverContext) + ); + List tests = new ArrayList<>(); + while (cases.isEmpty() == false) { + List b = randomNonEmptySubsetOf(cases); + cases.removeAll(b); + tests.addAll(b); + operators.add( + new ValuesSourceReaderOperator.Factory(b.stream().map(i -> i.info).toList(), shardContexts, 0).get(driverContext) + ); + } + List results = drive(operators, input.iterator(), driverContext); + assertThat(results, hasSize(input.size())); + for (Page page : results) { + assertThat(page.getBlockCount(), equalTo(tests.size() + 3 /* one for doc, one for keys and one for indexKey */)); + IntVector keys = page.getBlock(1).asVector(); + BytesRefVector indexKeys = page.getBlock(2).asVector(); + for (int p = 0; p < page.getPositionCount(); p++) { + int key = keys.getInt(p); + String indexKey = indexKeys.getBytesRef(p, new BytesRef()).utf8ToString(); + for (int i = 0; i < tests.size(); i++) { + try { + tests.get(i).checkResults.check(page.getBlock(3 + i), p, key, indexKey); + } catch (AssertionError e) { + throw new AssertionError("error checking " + tests.get(i).info.name() + "[" + p + "]: " + e.getMessage(), e); + } + } + } + } + for (Operator op : operators) { + assertThat(((ValuesSourceReaderOperator) op).status().pagesProcessed(), equalTo(input.size())); + } + assertDriverContext(driverContext); + } + + interface CheckResults { + void check(Block block, int position, int key, String indexKey); + } + + interface CheckReaders { + void check(boolean forcedRowByRow, int pageCount, int segmentCount, Map readersBuilt); + } + + interface CheckReadersWithName { + void check(String name, boolean forcedRowByRow, int pageCount, int segmentCount, Map readersBuilt); + } + + record FieldCase(ValuesSourceReaderOperator.FieldInfo info, CheckResults checkResults, CheckReadersWithName checkReaders) { + FieldCase(MappedFieldType ft, ElementType elementType, CheckResults checkResults, CheckReadersWithName checkReaders) { + this(fieldInfo(ft, elementType), checkResults, checkReaders); + } + + FieldCase( + MappedFieldType ft, + MappedFieldType ftX, + ElementType elementType, + CheckResults checkResults, + CheckReadersWithName checkReaders + ) { + this(fieldInfo(ft, ftX, elementType), checkResults, checkReaders); + } + + FieldCase(MappedFieldType ft, ElementType elementType, CheckResults checkResults, CheckReaders checkReaders) { + this( + ft, + elementType, + checkResults, + (name, forcedRowByRow, pageCount, segmentCount, readersBuilt) -> checkReaders.check( + forcedRowByRow, + pageCount, + segmentCount, + readersBuilt + ) + ); + } + } + + /** + * Asserts that {@link ValuesSourceReaderOperator#status} claims that only + * the expected readers are built after loading singleton pages. + */ + public void testLoadAllStatus() { + testLoadAllStatus(false); + } + + /** + * Asserts that {@link ValuesSourceReaderOperator#status} claims that only + * the expected readers are built after loading non-singleton pages. + */ + public void testLoadAllStatusAllInOnePage() { + testLoadAllStatus(true); + } + + private void testLoadAllStatus(boolean allInOnePage) { + DriverContext driverContext = driverContext(); + int numDocs = between(100, 5000); + List input = CannedSourceOperator.collectPages(simpleInput(driverContext, numDocs, commitEvery(numDocs), numDocs)); + assertThat(input, hasSize(20)); + List shardContexts = initShardContexts(); + int totalSize = 0; + for (var shardContext : shardContexts) { + assertThat(shardContext.reader().leaves(), hasSize(10)); + totalSize += shardContext.reader().leaves().size(); + } + // Build one operator for each field, so we get a unique map to assert on + List cases = infoAndChecksForEachType( + Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING, + Block.MvOrdering.DEDUPLICATED_AND_SORTED_ASCENDING + ); + List operators = cases.stream() + .map(i -> new ValuesSourceReaderOperator.Factory(List.of(i.info), shardContexts, 0).get(driverContext)) + .toList(); + if (allInOnePage) { + input = List.of(CannedSourceOperator.mergePages(input)); + } + drive(operators, input.iterator(), driverContext); + for (int i = 0; i < cases.size(); i++) { + ValuesSourceReaderOperator.Status status = (ValuesSourceReaderOperator.Status) operators.get(i).status(); + assertThat(status.pagesProcessed(), equalTo(input.size())); + FieldCase fc = cases.get(i); + fc.checkReaders.check(fc.info.name(), allInOnePage, input.size(), totalSize, status.readersBuilt()); + } + } + + private List infoAndChecksForEachType( + Block.MvOrdering booleanAndNumericalDocValuesMvOrdering, + Block.MvOrdering bytesRefDocValuesMvOrdering + ) { + MapperService mapperService = mapperService("index1"); // almost fields have identical mapper service + Checks checks = new Checks(booleanAndNumericalDocValuesMvOrdering, bytesRefDocValuesMvOrdering); + List r = new ArrayList<>(); + r.add(new FieldCase(mapperService.fieldType(IdFieldMapper.NAME), ElementType.BYTES_REF, checks::ids, StatusChecks::id)); + r.add(new FieldCase(TsidExtractingIdFieldMapper.INSTANCE.fieldType(), ElementType.BYTES_REF, checks::ids, StatusChecks::id)); + r.add(new FieldCase(mapperService.fieldType("long"), ElementType.LONG, checks::longs, StatusChecks::longsFromDocValues)); + r.add( + new FieldCase( + mapperService.fieldType("str_long"), + mapperService.fieldType("long"), + ElementType.LONG, + checks::longs, + StatusChecks::strFromDocValues + ) + ); + r.add( + new FieldCase( + mapperService.fieldType("mv_long"), + ElementType.LONG, + checks::mvLongsFromDocValues, + StatusChecks::mvLongsFromDocValues + ) + ); + r.add(new FieldCase(mapperService.fieldType("missing_long"), ElementType.LONG, checks::constantNulls, StatusChecks::constantNulls)); + r.add(new FieldCase(mapperService.fieldType("source_long"), ElementType.LONG, checks::longs, StatusChecks::longsFromSource)); + r.add( + new FieldCase( + mapperService.fieldType("mv_source_long"), + ElementType.LONG, + checks::mvLongsUnordered, + StatusChecks::mvLongsFromSource + ) + ); + r.add(new FieldCase(mapperService.fieldType("int"), ElementType.INT, checks::ints, StatusChecks::intsFromDocValues)); + r.add( + new FieldCase( + mapperService.fieldType("str_int"), + mapperService.fieldType("int"), + ElementType.INT, + checks::ints, + StatusChecks::strFromDocValues + ) + ); + r.add( + new FieldCase( + mapperService.fieldType("mv_int"), + ElementType.INT, + checks::mvIntsFromDocValues, + StatusChecks::mvIntsFromDocValues + ) + ); + r.add(new FieldCase(mapperService.fieldType("missing_int"), ElementType.INT, checks::constantNulls, StatusChecks::constantNulls)); + r.add(new FieldCase(mapperService.fieldType("source_int"), ElementType.INT, checks::ints, StatusChecks::intsFromSource)); + r.add( + new FieldCase( + mapperService.fieldType("mv_source_int"), + ElementType.INT, + checks::mvIntsUnordered, + StatusChecks::mvIntsFromSource + ) + ); + r.add(new FieldCase(mapperService.fieldType("short"), ElementType.INT, checks::shorts, StatusChecks::shortsFromDocValues)); + r.add( + new FieldCase( + mapperService.fieldType("str_short"), + mapperService.fieldType("short"), + ElementType.INT, + checks::shorts, + StatusChecks::strFromDocValues + ) + ); + r.add(new FieldCase(mapperService.fieldType("mv_short"), ElementType.INT, checks::mvShorts, StatusChecks::mvShortsFromDocValues)); + r.add(new FieldCase(mapperService.fieldType("missing_short"), ElementType.INT, checks::constantNulls, StatusChecks::constantNulls)); + r.add(new FieldCase(mapperService.fieldType("byte"), ElementType.INT, checks::bytes, StatusChecks::bytesFromDocValues)); + // r.add(new FieldCase(mapperService.fieldType("str_byte"), ElementType.INT, checks::bytes, StatusChecks::bytesFromDocValues)); + r.add(new FieldCase(mapperService.fieldType("mv_byte"), ElementType.INT, checks::mvBytes, StatusChecks::mvBytesFromDocValues)); + r.add(new FieldCase(mapperService.fieldType("missing_byte"), ElementType.INT, checks::constantNulls, StatusChecks::constantNulls)); + r.add(new FieldCase(mapperService.fieldType("double"), ElementType.DOUBLE, checks::doubles, StatusChecks::doublesFromDocValues)); + r.add( + new FieldCase( + mapperService.fieldType("str_double"), + mapperService.fieldType("double"), + ElementType.DOUBLE, + checks::doubles, + StatusChecks::strFromDocValues + ) + ); + r.add( + new FieldCase(mapperService.fieldType("mv_double"), ElementType.DOUBLE, checks::mvDoubles, StatusChecks::mvDoublesFromDocValues) + ); + r.add( + new FieldCase(mapperService.fieldType("missing_double"), ElementType.DOUBLE, checks::constantNulls, StatusChecks::constantNulls) + ); + r.add(new FieldCase(mapperService.fieldType("kwd"), ElementType.BYTES_REF, checks::tags, StatusChecks::keywordsFromDocValues)); + r.add( + new FieldCase( + mapperService.fieldType("mv_kwd"), + ElementType.BYTES_REF, + checks::mvStringsFromDocValues, + StatusChecks::mvKeywordsFromDocValues + ) + ); + r.add( + new FieldCase(mapperService.fieldType("missing_kwd"), ElementType.BYTES_REF, checks::constantNulls, StatusChecks::constantNulls) + ); + r.add(new FieldCase(storedKeywordField("stored_kwd"), ElementType.BYTES_REF, checks::strings, StatusChecks::keywordsFromStored)); + r.add( + new FieldCase( + storedKeywordField("mv_stored_kwd"), + ElementType.BYTES_REF, + checks::mvStringsUnordered, + StatusChecks::mvKeywordsFromStored + ) + ); + r.add( + new FieldCase(mapperService.fieldType("source_kwd"), ElementType.BYTES_REF, checks::strings, StatusChecks::keywordsFromSource) + ); + r.add( + new FieldCase( + mapperService.fieldType("mv_source_kwd"), + ElementType.BYTES_REF, + checks::mvStringsUnordered, + StatusChecks::mvKeywordsFromSource + ) + ); + r.add( + new FieldCase( + new ValuesSourceReaderOperator.FieldInfo( + "constant_bytes", + ElementType.BYTES_REF, + shardIdx -> BlockLoader.constantBytes(new BytesRef("foo")) + ), + checks::constantBytes, + StatusChecks::constantBytes + ) + ); + r.add( + new FieldCase( + new ValuesSourceReaderOperator.FieldInfo("null", ElementType.NULL, shardIdx -> BlockLoader.CONSTANT_NULLS), + checks::constantNulls, + StatusChecks::constantNulls + ) + ); + + // We only care about the field name at this point, so we can use any index mapper here + TestIndexMappingConfig indexMappingConfig = INDICES.get("index1"); + for (TestFieldType fieldType : indexMappingConfig.fieldTypes.values()) { + r.add( + new FieldCase( + fieldInfo(fieldType.name, ElementType.BYTES_REF, DataType.KEYWORD), + fieldType.checkResults, + StatusChecks::unionFromDocValues + ) + ); + } + Collections.shuffle(r, random()); + return r; + } + + record Checks(Block.MvOrdering booleanAndNumericalDocValuesMvOrdering, Block.MvOrdering bytesRefDocValuesMvOrdering) { + void longs(Block block, int position, int key, String indexKey) { + LongVector longs = ((LongBlock) block).asVector(); + assertThat(longs.getLong(position), equalTo((long) key)); + } + + void ints(Block block, int position, int key, String indexKey) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo(key)); + } + + void shorts(Block block, int position, int key, String indexKey) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo((int) (short) key)); + } + + void bytes(Block block, int position, int key, String indexKey) { + IntVector ints = ((IntBlock) block).asVector(); + assertThat(ints.getInt(position), equalTo((int) (byte) key)); + } + + void doubles(Block block, int position, int key, String indexKey) { + DoubleVector doubles = ((DoubleBlock) block).asVector(); + assertThat(doubles.getDouble(position), equalTo(key / 123_456d)); + } + + void strings(Block block, int position, int key, String indexKey) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + assertThat(keywords.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo(Integer.toString(key))); + } + + static void unionIPsAsStrings(Block block, int position, int key, String indexKey) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + BytesRef bytesRef = keywords.getBytesRef(position, new BytesRef()); + TestIndexMappingConfig mappingConfig = INDICES.get(indexKey); + TestFieldType fieldType = mappingConfig.fieldTypes.get("ip"); + String expected = fieldType.valueGenerator.apply(key).toString(); + // Conversion should already be done in FieldInfo! + // BytesRef found = (fieldType.dataType.typeName().equals("ip")) ? new BytesRef(DocValueFormat.IP.format(bytesRef)) : bytesRef; + assertThat(bytesRef.utf8ToString(), equalTo(expected)); + } + + static void unionDurationsAsStrings(Block block, int position, int key, String indexKey) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + BytesRef bytesRef = keywords.getBytesRef(position, new BytesRef()); + TestIndexMappingConfig mappingConfig = INDICES.get(indexKey); + TestFieldType fieldType = mappingConfig.fieldTypes.get("duration"); + String expected = fieldType.valueGenerator.apply(key).toString(); + assertThat(bytesRef.utf8ToString(), equalTo(expected)); + } + + void tags(Block block, int position, int key, String indexKey) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + Object[] validTags = INDICES.keySet().stream().map(keyToTags::get).map(t -> t.get(key)).toArray(); + assertThat(keywords.getBytesRef(position, new BytesRef()).utf8ToString(), oneOf(validTags)); + } + + void ids(Block block, int position, int key, String indexKey) { + BytesRefVector ids = ((BytesRefBlock) block).asVector(); + assertThat(ids.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo("id" + key)); + } + + void constantBytes(Block block, int position, int key, String indexKey) { + BytesRefVector keywords = ((BytesRefBlock) block).asVector(); + assertThat(keywords.getBytesRef(position, new BytesRef()).utf8ToString(), equalTo("foo")); + } + + void constantNulls(Block block, int position, int key, String indexKey) { + assertTrue(block.areAllValuesNull()); + assertTrue(block.isNull(position)); + } + + void mvLongsFromDocValues(Block block, int position, int key, String indexKey) { + mvLongs(block, position, key, booleanAndNumericalDocValuesMvOrdering); + } + + void mvLongsUnordered(Block block, int position, int key, String indexKey) { + mvLongs(block, position, key, Block.MvOrdering.UNORDERED); + } + + private void mvLongs(Block block, int position, int key, Block.MvOrdering expectedMv) { + LongBlock longs = (LongBlock) block; + assertThat(longs.getValueCount(position), equalTo(key % 3 + 1)); + int offset = longs.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(longs.getLong(offset + v), equalTo(-1_000L * key + v)); + } + if (key % 3 > 0) { + assertThat(longs.mvOrdering(), equalTo(expectedMv)); + } + } + + void mvIntsFromDocValues(Block block, int position, int key, String indexKey) { + mvInts(block, position, key, booleanAndNumericalDocValuesMvOrdering); + } + + void mvIntsUnordered(Block block, int position, int key, String indexKey) { + mvInts(block, position, key, Block.MvOrdering.UNORDERED); + } + + private void mvInts(Block block, int position, int key, Block.MvOrdering expectedMv) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo(1_000 * key + v)); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(expectedMv)); + } + } + + void mvShorts(Block block, int position, int key, String indexKey) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo((int) (short) (2_000 * key + v))); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(booleanAndNumericalDocValuesMvOrdering)); + } + } + + void mvBytes(Block block, int position, int key, String indexKey) { + IntBlock ints = (IntBlock) block; + assertThat(ints.getValueCount(position), equalTo(key % 3 + 1)); + int offset = ints.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(ints.getInt(offset + v), equalTo((int) (byte) (3_000 * key + v))); + } + if (key % 3 > 0) { + assertThat(ints.mvOrdering(), equalTo(booleanAndNumericalDocValuesMvOrdering)); + } + } + + void mvDoubles(Block block, int position, int key, String indexKey) { + DoubleBlock doubles = (DoubleBlock) block; + int offset = doubles.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(doubles.getDouble(offset + v), equalTo(key / 123_456d + v)); + } + if (key % 3 > 0) { + assertThat(doubles.mvOrdering(), equalTo(booleanAndNumericalDocValuesMvOrdering)); + } + } + + void mvStringsFromDocValues(Block block, int position, int key, String indexKey) { + mvStrings(block, position, key, bytesRefDocValuesMvOrdering); + } + + void mvStringsUnordered(Block block, int position, int key, String indexKey) { + mvStrings(block, position, key, Block.MvOrdering.UNORDERED); + } + + void mvStrings(Block block, int position, int key, Block.MvOrdering expectedMv) { + BytesRefBlock text = (BytesRefBlock) block; + assertThat(text.getValueCount(position), equalTo(key % 3 + 1)); + int offset = text.getFirstValueIndex(position); + for (int v = 0; v <= key % 3; v++) { + assertThat(text.getBytesRef(offset + v, new BytesRef()).utf8ToString(), equalTo(PREFIX[v] + key)); + } + if (key % 3 > 0) { + assertThat(text.mvOrdering(), equalTo(expectedMv)); + } + } + } + + static class StatusChecks { + + static void strFromDocValues(String name, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues(name, "Ordinals", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void longsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void longsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void intsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void intsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void shortsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("short", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void bytesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("byte", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void doublesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("double", "Doubles", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + docValues("kwd", "Ordinals", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("stored_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void keywordsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("source_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvLongsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvLongsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_long", "Longs", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvIntsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvIntsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_int", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvShortsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_short", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvBytesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_byte", "Ints", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvDoublesFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_double", "Doubles", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromDocValues(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + mvDocValues("mv_kwd", "Ordinals", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromStored(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("mv_stored_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void mvKeywordsFromSource(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + source("mv_source_kwd", "Bytes", forcedRowByRow, pageCount, segmentCount, readers); + } + + static void unionFromDocValues(String name, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + // TODO: develop a working check for this + // docValues(name, "Ordinals", forcedRowByRow, pageCount, segmentCount, readers); + } + + private static void docValues( + String name, + String type, + boolean forcedRowByRow, + int pageCount, + int segmentCount, + Map readers + ) { + if (forcedRowByRow) { + assertMap( + "Expected segment count in " + readers + "\n", + readers, + matchesMap().entry(name + ":row_stride:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(segmentCount)) + ); + } else { + assertMap( + "Expected segment count in " + readers + "\n", + readers, + matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type, lessThanOrEqualTo(pageCount)) + ); + } + } + + private static void mvDocValues( + String name, + String type, + boolean forcedRowByRow, + int pageCount, + int segmentCount, + Map readers + ) { + if (forcedRowByRow) { + Integer singletons = (Integer) readers.remove(name + ":row_stride:BlockDocValuesReader.Singleton" + type); + if (singletons != null) { + segmentCount -= singletons; + } + assertMap(readers, matchesMap().entry(name + ":row_stride:BlockDocValuesReader." + type, segmentCount)); + } else { + Integer singletons = (Integer) readers.remove(name + ":column_at_a_time:BlockDocValuesReader.Singleton" + type); + if (singletons != null) { + pageCount -= singletons; + } + assertMap( + readers, + matchesMap().entry(name + ":column_at_a_time:BlockDocValuesReader." + type, lessThanOrEqualTo(pageCount)) + ); + } + } + + static void id(boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + stored("_id", "Id", forcedRowByRow, pageCount, segmentCount, readers); + } + + private static void source(String name, String type, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + Matcher count; + if (forcedRowByRow) { + count = equalTo(segmentCount); + } else { + count = lessThanOrEqualTo(pageCount); + Integer columnAttempts = (Integer) readers.remove(name + ":column_at_a_time:null"); + assertThat(columnAttempts, not(nullValue())); + } + + Integer sequentialCount = (Integer) readers.remove("stored_fields[requires_source:true, fields:0, sequential: true]"); + Integer nonSequentialCount = (Integer) readers.remove("stored_fields[requires_source:true, fields:0, sequential: false]"); + int totalReaders = (sequentialCount == null ? 0 : sequentialCount) + (nonSequentialCount == null ? 0 : nonSequentialCount); + assertThat(totalReaders, count); + + assertMap(readers, matchesMap().entry(name + ":row_stride:BlockSourceReader." + type, count)); + } + + private static void stored(String name, String type, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + Matcher count; + if (forcedRowByRow) { + count = equalTo(segmentCount); + } else { + count = lessThanOrEqualTo(pageCount); + Integer columnAttempts = (Integer) readers.remove(name + ":column_at_a_time:null"); + assertThat(columnAttempts, not(nullValue())); + } + + Integer sequentialCount = (Integer) readers.remove("stored_fields[requires_source:false, fields:1, sequential: true]"); + Integer nonSequentialCount = (Integer) readers.remove("stored_fields[requires_source:false, fields:1, sequential: false]"); + int totalReaders = (sequentialCount == null ? 0 : sequentialCount) + (nonSequentialCount == null ? 0 : nonSequentialCount); + assertThat(totalReaders, count); + + assertMap(readers, matchesMap().entry(name + ":row_stride:BlockStoredFieldsReader." + type, count)); + } + + static void constantBytes(String name, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap(readers, matchesMap().entry(name + ":row_stride:constant[[66 6f 6f]]", segmentCount)); + } else { + assertMap(readers, matchesMap().entry(name + ":column_at_a_time:constant[[66 6f 6f]]", lessThanOrEqualTo(pageCount))); + } + } + + static void constantNulls(String name, boolean forcedRowByRow, int pageCount, int segmentCount, Map readers) { + if (forcedRowByRow) { + assertMap(readers, matchesMap().entry(name + ":row_stride:constant_nulls", segmentCount)); + } else { + assertMap(readers, matchesMap().entry(name + ":column_at_a_time:constant_nulls", lessThanOrEqualTo(pageCount))); + } + } + } + + public void testWithNulls() throws IOException { + String indexKey = "index1"; + mapperServices.put(indexKey, new MapperServiceTestCase() { + }.createMapperService(MapperServiceTestCase.mapping(b -> { + fieldExamples(b, "i", "integer"); + fieldExamples(b, "j", "long"); + fieldExamples(b, "d", "double"); + }))); + MappedFieldType intFt = mapperService(indexKey).fieldType("i"); + MappedFieldType longFt = mapperService(indexKey).fieldType("j"); + MappedFieldType doubleFt = mapperService(indexKey).fieldType("d"); + MappedFieldType kwFt = new KeywordFieldMapper.KeywordFieldType("kw"); + + NumericDocValuesField intField = new NumericDocValuesField(intFt.name(), 0); + NumericDocValuesField longField = new NumericDocValuesField(longFt.name(), 0); + NumericDocValuesField doubleField = new DoubleDocValuesField(doubleFt.name(), 0); + final int numDocs = between(100, 5000); + try (RandomIndexWriter w = new RandomIndexWriter(random(), directory(indexKey))) { + Document doc = new Document(); + for (int i = 0; i < numDocs; i++) { + doc.clear(); + intField.setLongValue(i); + doc.add(intField); + if (i % 100 != 0) { // Do not set field for every 100 values + longField.setLongValue(i); + doc.add(longField); + doubleField.setDoubleValue(i); + doc.add(doubleField); + doc.add(new SortedDocValuesField(kwFt.name(), new BytesRef("kw=" + i))); + } + w.addDocument(doc); + } + w.commit(); + readers.put(indexKey, w.getReader()); + } + LuceneSourceOperatorTests.MockShardContext shardContext = new LuceneSourceOperatorTests.MockShardContext(reader(indexKey), 0); + DriverContext driverContext = driverContext(); + var luceneFactory = new LuceneSourceOperator.Factory( + List.of(shardContext), + ctx -> new MatchAllDocsQuery(), + randomFrom(DataPartitioning.values()), + randomIntBetween(1, 10), + randomPageSize(), + LuceneOperator.NO_LIMIT + ); + var vsShardContext = new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE); + try ( + Driver driver = new Driver( + driverContext, + luceneFactory.get(driverContext), + List.of( + factory(List.of(vsShardContext), intFt, ElementType.INT).get(driverContext), + factory(List.of(vsShardContext), longFt, ElementType.LONG).get(driverContext), + factory(List.of(vsShardContext), doubleFt, ElementType.DOUBLE).get(driverContext), + factory(List.of(vsShardContext), kwFt, ElementType.BYTES_REF).get(driverContext) + ), + new PageConsumerOperator(page -> { + try { + logger.debug("New page: {}", page); + IntBlock intValuesBlock = page.getBlock(1); + LongBlock longValuesBlock = page.getBlock(2); + DoubleBlock doubleValuesBlock = page.getBlock(3); + BytesRefBlock keywordValuesBlock = page.getBlock(4); + + for (int i = 0; i < page.getPositionCount(); i++) { + assertFalse(intValuesBlock.isNull(i)); + long j = intValuesBlock.getInt(i); + // Every 100 documents we set fields to null + boolean fieldIsEmpty = j % 100 == 0; + assertEquals(fieldIsEmpty, longValuesBlock.isNull(i)); + assertEquals(fieldIsEmpty, doubleValuesBlock.isNull(i)); + assertEquals(fieldIsEmpty, keywordValuesBlock.isNull(i)); + } + } finally { + page.releaseBlocks(); + } + }), + () -> {} + ) + ) { + runDriver(driver); + } + assertDriverContext(driverContext); + } + + private XContentBuilder fieldExamples(XContentBuilder builder, String name, String type) throws IOException { + simpleField(builder, name, type); + simpleField(builder, "str_" + name, "keyword"); + simpleField(builder, "mv_" + name, type); + simpleField(builder, "missing_" + name, type); + sourceField(builder, "source_" + name, type); + return sourceField(builder, "mv_source_" + name, type); + } + + private XContentBuilder simpleField(XContentBuilder builder, String name, String type) throws IOException { + return builder.startObject(name).field("type", type).endObject(); + } + + private XContentBuilder sourceField(XContentBuilder builder, String name, String type) throws IOException { + return builder.startObject(name).field("type", type).field("store", false).field("doc_values", false).endObject(); + } + + private KeywordFieldMapper.KeywordFieldType storedKeywordField(String name) { + FieldType ft = new FieldType(KeywordFieldMapper.Defaults.FIELD_TYPE); + ft.setDocValuesType(DocValuesType.NONE); + ft.setStored(true); + ft.freeze(); + return new KeywordFieldMapper.KeywordFieldType( + name, + ft, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER, + new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(false), + true // TODO randomize - load from stored keyword fields if stored even in synthetic source + ); + } + + @AwaitsFix(bugUrl = "Get working for multiple indices") + public void testNullsShared() { + DriverContext driverContext = driverContext(); + List shardContexts = initShardContexts(); + int[] pages = new int[] { 0 }; + try ( + Driver d = new Driver( + driverContext, + simpleInput(driverContext, 10), + List.of( + new ValuesSourceReaderOperator.Factory( + List.of( + new ValuesSourceReaderOperator.FieldInfo("null1", ElementType.NULL, shardIdx -> BlockLoader.CONSTANT_NULLS), + new ValuesSourceReaderOperator.FieldInfo("null2", ElementType.NULL, shardIdx -> BlockLoader.CONSTANT_NULLS) + ), + shardContexts, + 0 + ).get(driverContext) + ), + new PageConsumerOperator(page -> { + try { + assertThat(page.getBlockCount(), equalTo(3)); + assertThat(page.getBlock(1).areAllValuesNull(), equalTo(true)); + assertThat(page.getBlock(2).areAllValuesNull(), equalTo(true)); + assertThat(page.getBlock(1), sameInstance(page.getBlock(2))); + pages[0]++; + } finally { + page.releaseBlocks(); + } + }), + () -> {} + ) + ) { + runDriver(d); + } + assertThat(pages[0], greaterThan(0)); + assertDriverContext(driverContext); + } + + public void testDescriptionOfMany() throws IOException { + String indexKey = "index1"; + initIndex(indexKey, 1, 1); + Block.MvOrdering ordering = randomFrom(Block.MvOrdering.values()); + List cases = infoAndChecksForEachType(ordering, ordering); + + ValuesSourceReaderOperator.Factory factory = new ValuesSourceReaderOperator.Factory( + cases.stream().map(c -> c.info).toList(), + List.of(new ValuesSourceReaderOperator.ShardContext(reader(indexKey), () -> SourceLoader.FROM_STORED_SOURCE)), + 0 + ); + assertThat(factory.describe(), equalTo("ValuesSourceReaderOperator[fields = [" + cases.size() + " fields]]")); + try (Operator op = factory.get(driverContext())) { + assertThat(op.toString(), equalTo("ValuesSourceReaderOperator[fields = [" + cases.size() + " fields]]")); + } + } + + public void testManyShards() throws IOException { + String indexKey = "index1"; + initMapping(indexKey); + int shardCount = between(2, 10); + int size = between(100, 1000); + Directory[] dirs = new Directory[shardCount]; + IndexReader[] readers = new IndexReader[shardCount]; + Closeable[] closeMe = new Closeable[shardCount * 2]; + Set seenShards = new TreeSet<>(); + Map keyCounts = new TreeMap<>(); + try { + for (int d = 0; d < dirs.length; d++) { + closeMe[d * 2 + 1] = dirs[d] = newDirectory(); + closeMe[d * 2] = readers[d] = initIndex(indexKey, dirs[d], size, between(10, size * 2)); + } + List contexts = new ArrayList<>(); + List readerShardContexts = new ArrayList<>(); + for (int s = 0; s < shardCount; s++) { + contexts.add(new LuceneSourceOperatorTests.MockShardContext(readers[s], s)); + readerShardContexts.add(new ValuesSourceReaderOperator.ShardContext(readers[s], () -> SourceLoader.FROM_STORED_SOURCE)); + } + var luceneFactory = new LuceneSourceOperator.Factory( + contexts, + ctx -> new MatchAllDocsQuery(), + DataPartitioning.SHARD, + randomIntBetween(1, 10), + 1000, + LuceneOperator.NO_LIMIT + ); + // TODO add index2 + MappedFieldType ft = mapperService(indexKey).fieldType("key"); + var readerFactory = new ValuesSourceReaderOperator.Factory( + List.of(new ValuesSourceReaderOperator.FieldInfo("key", ElementType.INT, shardIdx -> { + seenShards.add(shardIdx); + return ft.blockLoader(blContext()); + })), + readerShardContexts, + 0 + ); + DriverContext driverContext = driverContext(); + List results = drive( + readerFactory.get(driverContext), + CannedSourceOperator.collectPages(luceneFactory.get(driverContext)).iterator(), + driverContext + ); + assertThat(seenShards, equalTo(IntStream.range(0, shardCount).boxed().collect(Collectors.toCollection(TreeSet::new)))); + for (Page p : results) { + IntBlock keyBlock = p.getBlock(1); + IntVector keys = keyBlock.asVector(); + for (int i = 0; i < keys.getPositionCount(); i++) { + keyCounts.merge(keys.getInt(i), 1, Integer::sum); + } + } + assertThat(keyCounts.keySet(), hasSize(size)); + for (int k = 0; k < size; k++) { + assertThat(keyCounts.get(k), equalTo(shardCount)); + } + } finally { + IOUtils.close(closeMe); + } + } + + protected final List drive(Operator operator, Iterator input, DriverContext driverContext) { + return drive(List.of(operator), input, driverContext); + } + + protected final List drive(List operators, Iterator input, DriverContext driverContext) { + List results = new ArrayList<>(); + boolean success = false; + try ( + Driver d = new Driver( + driverContext, + new CannedSourceOperator(input), + operators, + new TestResultPageSinkOperator(results::add), + () -> {} + ) + ) { + runDriver(d); + success = true; + } finally { + if (success == false) { + Releasables.closeExpectNoException(Releasables.wrap(() -> Iterators.map(results.iterator(), p -> p::releaseBlocks))); + } + } + return results; + } + + public static void runDriver(Driver driver) { + runDriver(List.of(driver)); + } + + public static void runDriver(List drivers) { + drivers = new ArrayList<>(drivers); + int dummyDrivers = between(0, 10); + for (int i = 0; i < dummyDrivers; i++) { + drivers.add( + new Driver( + "dummy-session", + 0, + 0, + new DriverContext(BigArrays.NON_RECYCLING_INSTANCE, TestBlockFactory.getNonBreakingInstance()), + () -> "dummy-driver", + new SequenceLongBlockSourceOperator( + TestBlockFactory.getNonBreakingInstance(), + LongStream.range(0, between(1, 100)), + between(1, 100) + ), + List.of(), + new PageConsumerOperator(Page::releaseBlocks), + Driver.DEFAULT_STATUS_INTERVAL, + () -> {} + ) + ); + } + Randomness.shuffle(drivers); + int numThreads = between(1, 16); + ThreadPool threadPool = new TestThreadPool( + getTestClass().getSimpleName(), + new FixedExecutorBuilder(Settings.EMPTY, "esql", numThreads, 1024, "esql", EsExecutors.TaskTrackingConfig.DEFAULT) + ); + var driverRunner = new DriverRunner(threadPool.getThreadContext()) { + @Override + protected void start(Driver driver, ActionListener driverListener) { + Driver.start(threadPool.getThreadContext(), threadPool.executor("esql"), driver, between(1, 10000), driverListener); + } + }; + PlainActionFuture future = new PlainActionFuture<>(); + try { + driverRunner.runToCompletion(drivers, future); + future.actionGet(TimeValue.timeValueSeconds(30)); + } finally { + terminate(threadPool); + } + } + + public static void assertDriverContext(DriverContext driverContext) { + assertTrue(driverContext.isFinished()); + assertThat(driverContext.getSnapshot().releasables(), empty()); + } + + public static int randomPageSize() { + if (randomBoolean()) { + return between(1, 16); + } else { + return between(1, 16 * 1024); + } + } + + /** + * This method will produce the same converter for all shards, which makes it useful for general type converting tests, + * but not specifically union-types tests which require different converters for each shard. + */ + private static BlockLoader getBlockLoaderFor(int shardIdx, MappedFieldType ft, MappedFieldType ftX) { + if (shardIdx < 0 || shardIdx >= INDICES.size()) { + fail("unexpected shardIdx [" + shardIdx + "]"); + } + BlockLoader blockLoader = ft.blockLoader(blContext()); + if (ftX != null && ftX.typeName().equals(ft.typeName()) == false) { + blockLoader = new TestTypeConvertingBlockLoader(blockLoader, ft.typeName(), ftX.typeName()); + } else { + TestIndexMappingConfig mappingConfig = INDICES.get("index" + (shardIdx + 1)); + TestFieldType testFieldType = mappingConfig.fieldTypes.get(ft.name()); + if (testFieldType != null) { + blockLoader = new TestTypeConvertingBlockLoader(blockLoader, testFieldType.dataType.typeName(), "keyword"); + } + } + return blockLoader; + } + + /** + * This method is used to generate shard-specific field information, so we can have different types and BlockLoaders for each shard. + */ + private BlockLoader getBlockLoaderFor(int shardIdx, String fieldName, DataType toType) { + if (shardIdx < 0 || shardIdx >= INDICES.size()) { + fail("unexpected shardIdx [" + shardIdx + "]"); + } + String indexKey = "index" + (shardIdx + 1); + TestIndexMappingConfig mappingConfig = INDICES.get(indexKey); + TestFieldType testFieldType = mappingConfig.fieldTypes.get(fieldName); + if (testFieldType == null) { + throw new IllegalArgumentException("Unknown test field: " + fieldName); + } + MapperService mapper = mapperService(indexKey); + MappedFieldType ft = mapper.fieldType(fieldName); + BlockLoader blockLoader = ft.blockLoader(blContext()); + blockLoader = new TestTypeConvertingBlockLoader(blockLoader, testFieldType.dataType.typeName(), toType.typeName()); + return blockLoader; + } + + /** + * The implementation of union-types relies on the BlockLoader.convert(Block) to convert the block to the correct type + * at the point it is read from source, so that the rest of the query only deals with a single type for that field. + * This is implemented in the 'esql' module, and so we have a mock for this behaviour here, which is a simplified subset of the + * features in the real implementation. + */ + static class TestTypeConvertingBlockLoader implements BlockLoader { + protected final BlockLoader delegate; + private final EvalOperator.ExpressionEvaluator convertEvaluator; + + protected TestTypeConvertingBlockLoader(BlockLoader delegate, String fromTypeName, String toTypeName) { + this.delegate = delegate; + DriverContext driverContext = new DriverContext( + BigArrays.NON_RECYCLING_INSTANCE, + new org.elasticsearch.compute.data.BlockFactory( + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + BigArrays.NON_RECYCLING_INSTANCE + ) + ); + TestBlockConverter blockConverter = TestDataTypeConverters.blockConverter(driverContext, fromTypeName, toTypeName); + this.convertEvaluator = new EvalOperator.ExpressionEvaluator() { + @Override + public org.elasticsearch.compute.data.Block eval(Page page) { + org.elasticsearch.compute.data.Block block = page.getBlock(0); + return blockConverter.convert(block); + } + + @Override + public void close() {} + }; + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + // Return the delegates builder, which can build the original mapped type, before conversion + return delegate.builder(factory, expectedCount); + } + + @Override + public Block convert(Block block) { + Page page = new Page((org.elasticsearch.compute.data.Block) block); + return convertEvaluator.eval(page); + } + + @Override + public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + ColumnAtATimeReader reader = delegate.columnAtATimeReader(context); + if (reader == null) { + return null; + } + return new ColumnAtATimeReader() { + @Override + public Block read(BlockFactory factory, Docs docs) throws IOException { + Block block = reader.read(factory, docs); + Page page = new Page((org.elasticsearch.compute.data.Block) block); + return convertEvaluator.eval(page); + } + + @Override + public boolean canReuse(int startingDocID) { + return reader.canReuse(startingDocID); + } + + @Override + public String toString() { + return reader.toString(); + } + }; + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + // We do no type conversion here, since that will be done in the ValueSourceReaderOperator for row-stride cases + // Using the BlockLoader.convert(Block) function defined above + return delegate.rowStrideReader(context); + } + + @Override + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return delegate.rowStrideStoredFieldSpec(); + } + + @Override + public boolean supportsOrdinals() { + return delegate.supportsOrdinals(); + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { + return delegate.ordinals(context); + } + + @Override + public final String toString() { + return "TypeConvertingBlockLoader[delegate=" + delegate + "]"; + } + } + + @FunctionalInterface + private interface TestBlockConverter { + Block convert(Block block); + } + + /** + * Blocks that should be converted from some type to a string (keyword) can use this converter. + */ + private abstract static class BlockToStringConverter implements TestBlockConverter { + private final DriverContext driverContext; + + BlockToStringConverter(DriverContext driverContext) { + this.driverContext = driverContext; + } + + @Override + public Block convert(Block block) { + int positionCount = block.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + BytesRef value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendBytesRef(value); + valuesAppended = true; + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } finally { + block.close(); + } + } + + abstract BytesRef evalValue(Block container, int index); + } + + /** + * Blocks that should be converted from a string (keyword) to some other type can use this converter. + */ + private abstract static class TestBlockFromStringConverter implements TestBlockConverter { + protected final DriverContext driverContext; + + TestBlockFromStringConverter(DriverContext driverContext) { + this.driverContext = driverContext; + } + + @Override + public Block convert(Block b) { + BytesRefBlock block = (BytesRefBlock) b; + int positionCount = block.getPositionCount(); + try (Block.Builder builder = blockBuilder(positionCount)) { + BytesRef scratchPad = new BytesRef(); + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + T value = evalValue(block, i, scratchPad); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + appendValue(builder, value); + valuesAppended = true; + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } finally { + b.close(); + } + } + + abstract Block.Builder blockBuilder(int expectedCount); + + abstract void appendValue(Block.Builder builder, T value); + + abstract T evalValue(BytesRefBlock container, int index, BytesRef scratchPad); + } + + private static class TestLongBlockToStringConverter extends BlockToStringConverter { + TestLongBlockToStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + BytesRef evalValue(Block container, int index) { + return new BytesRef(Long.toString(((LongBlock) container).getLong(index))); + } + } + + private static class TestLongBlockFromStringConverter extends TestBlockFromStringConverter { + TestLongBlockFromStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + Block.Builder blockBuilder(int expectedCount) { + return driverContext.blockFactory().newLongBlockBuilder(expectedCount); + } + + @Override + Long evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + return StringUtils.parseLong(container.getBytesRef(index, scratchPad).utf8ToString()); + } + + @Override + void appendValue(Block.Builder builder, Long value) { + ((LongBlock.Builder) builder).appendLong(value); + } + } + + private static class TestIntegerBlockToStringConverter extends BlockToStringConverter { + TestIntegerBlockToStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + BytesRef evalValue(Block container, int index) { + return new BytesRef(Integer.toString(((IntBlock) container).getInt(index))); + } + } + + private static class TestIntegerBlockFromStringConverter extends TestBlockFromStringConverter { + TestIntegerBlockFromStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + Block.Builder blockBuilder(int expectedCount) { + return driverContext.blockFactory().newIntBlockBuilder(expectedCount); + } + + @Override + Integer evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + return (int) StringUtils.parseLong(container.getBytesRef(index, scratchPad).utf8ToString()); + } + + @Override + void appendValue(Block.Builder builder, Integer value) { + ((IntBlock.Builder) builder).appendInt(value); + } + } + + private static class TestBooleanBlockToStringConverter extends BlockToStringConverter { + + TestBooleanBlockToStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + BytesRef evalValue(Block container, int index) { + return ((BooleanBlock) container).getBoolean(index) ? new BytesRef("true") : new BytesRef("false"); + } + } + + private static class TestBooleanBlockFromStringConverter extends TestBlockFromStringConverter { + + TestBooleanBlockFromStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + Block.Builder blockBuilder(int expectedCount) { + return driverContext.blockFactory().newBooleanBlockBuilder(expectedCount); + } + + @Override + void appendValue(Block.Builder builder, Boolean value) { + ((BooleanBlock.Builder) builder).appendBoolean(value); + } + + @Override + Boolean evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + return Boolean.parseBoolean(container.getBytesRef(index, scratchPad).utf8ToString()); + } + } + + private static class TestDoubleBlockToStringConverter extends BlockToStringConverter { + + TestDoubleBlockToStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + BytesRef evalValue(Block container, int index) { + return new BytesRef(Double.toString(((DoubleBlock) container).getDouble(index))); + } + } + + private static class TestDoubleBlockFromStringConverter extends TestBlockFromStringConverter { + + TestDoubleBlockFromStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + Block.Builder blockBuilder(int expectedCount) { + return driverContext.blockFactory().newDoubleBlockBuilder(expectedCount); + } + + @Override + void appendValue(Block.Builder builder, Double value) { + ((DoubleBlock.Builder) builder).appendDouble(value); + } + + @Override + Double evalValue(BytesRefBlock container, int index, BytesRef scratchPad) { + return Double.parseDouble(container.getBytesRef(index, scratchPad).utf8ToString()); + } + } + + /** + * Many types are backed by BytesRef block, but encode their contents in different ways. + * For example, the IP type has a 16-byte block that encodes both IPv4 and IPv6 as 16byte-IPv6 binary byte arrays. + * But the KEYWORD type has a BytesRef block that encodes the keyword as a UTF-8 string, + * and it typically has a much shorter length for IP data, for example, "192.168.0.1" is 11 bytes. + * Converting blocks between these types involves converting the BytesRef block to the specific internal type, + * and then back to a BytesRef block with the other encoding. + */ + private abstract static class TestBytesRefToBytesRefConverter extends BlockToStringConverter { + + BytesRef scratchPad = new BytesRef(); + + TestBytesRefToBytesRefConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + BytesRef evalValue(Block container, int index) { + return convertByteRef(((BytesRefBlock) container).getBytesRef(index, scratchPad)); + } + + abstract BytesRef convertByteRef(BytesRef bytesRef); + } + + private static class TestIPToStringConverter extends TestBytesRefToBytesRefConverter { + + TestIPToStringConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + BytesRef convertByteRef(BytesRef bytesRef) { + return new BytesRef(DocValueFormat.IP.format(bytesRef)); + } + } + + private static class TestStringToIPConverter extends TestBytesRefToBytesRefConverter { + + TestStringToIPConverter(DriverContext driverContext) { + super(driverContext); + } + + @Override + BytesRef convertByteRef(BytesRef bytesRef) { + return StringUtils.parseIP(bytesRef.utf8ToString()); + } + } + + /** + * Utility class for creating type-specific converters based on their typeNamne values. + * We do not support all possibly combinations, but only those that are needed for the tests. + * In particular, either the 'from' or 'to' types must be KEYWORD. + */ + private static class TestDataTypeConverters { + public static TestBlockConverter blockConverter(DriverContext driverContext, String fromTypeName, String toTypeName) { + if (toTypeName == null || fromTypeName.equals(toTypeName)) { + return b -> b; + } + if (isString(fromTypeName)) { + return switch (toTypeName) { + case "boolean" -> new TestBooleanBlockFromStringConverter(driverContext); + case "short", "integer" -> new TestIntegerBlockFromStringConverter(driverContext); + case "long" -> new TestLongBlockFromStringConverter(driverContext); + case "double", "float" -> new TestDoubleBlockFromStringConverter(driverContext); + case "ip" -> new TestStringToIPConverter(driverContext); + default -> throw new UnsupportedOperationException("Conversion from string to " + toTypeName + " is not supported"); + }; + } + if (isString(toTypeName)) { + return switch (fromTypeName) { + case "boolean" -> new TestBooleanBlockToStringConverter(driverContext); + case "short", "integer" -> new TestIntegerBlockToStringConverter(driverContext); + case "long" -> new TestLongBlockToStringConverter(driverContext); + case "double", "float" -> new TestDoubleBlockToStringConverter(driverContext); + case "ip" -> new TestIPToStringConverter(driverContext); + default -> throw new UnsupportedOperationException("Conversion from " + fromTypeName + " to string is not supported"); + }; + } + throw new UnsupportedOperationException("Conversion from " + fromTypeName + " to " + toTypeName + " is not supported"); + } + + private static boolean isString(String typeName) { + return typeName.equals("keyword") || typeName.equals("text"); + } + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java index 848f3750a4b20..fa72721545ab9 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/ForkingOperatorTestCase.java @@ -31,6 +31,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.Iterator; import java.util.List; import java.util.Set; @@ -353,4 +354,26 @@ public void setThreadPool() { public void shutdownThreadPool() { terminate(threadPool); } + + protected Comparator floatComparator() { + return FloatComparator.INSTANCE; + } + + static final class FloatComparator implements Comparator { + + static final FloatComparator INSTANCE = new FloatComparator(); + + @Override + public int compare(Float o1, Float o2) { + float first = o1; + float second = o2; + if (first < second) { + return -1; + } else if (first == second) { + return 0; + } else { + return 1; + } + } + } } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LongFloatTupleBlockSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LongFloatTupleBlockSourceOperator.java new file mode 100644 index 0000000000000..9276174c9dbb1 --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/LongFloatTupleBlockSourceOperator.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Tuple; + +import java.util.List; +import java.util.stream.Stream; + +/** + * A source operator whose output is the given tuple values. This operator produces pages + * with two Blocks. The returned pages preserve the order of values as given in the in initial list. + */ +public class LongFloatTupleBlockSourceOperator extends AbstractBlockSourceOperator { + + private static final int DEFAULT_MAX_PAGE_POSITIONS = 8 * 1024; + + private final List> values; + + public LongFloatTupleBlockSourceOperator(BlockFactory blockFactory, Stream> values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public LongFloatTupleBlockSourceOperator(BlockFactory blockFactory, Stream> values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + this.values = values.toList(); + } + + public LongFloatTupleBlockSourceOperator(BlockFactory blockFactory, List> values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public LongFloatTupleBlockSourceOperator(BlockFactory blockFactory, List> values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + this.values = values; + } + + @Override + protected Page createPage(int positionOffset, int length) { + var blockBuilder1 = blockFactory.newLongBlockBuilder(length); + var blockBuilder2 = blockFactory.newFloatBlockBuilder(length); + for (int i = 0; i < length; i++) { + Tuple item = values.get(positionOffset + i); + if (item.v1() == null) { + blockBuilder1.appendNull(); + } else { + blockBuilder1.appendLong(item.v1()); + } + if (item.v2() == null) { + blockBuilder2.appendNull(); + } else { + blockBuilder2.appendFloat(item.v2()); + } + } + currentPosition += length; + return new Page(blockBuilder1.build(), blockBuilder2.build()); + } + + @Override + protected int remaining() { + return values.size() - currentPosition; + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/NullInsertingSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/NullInsertingSourceOperator.java index 260918396dcd3..c8444551f415b 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/NullInsertingSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/NullInsertingSourceOperator.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -99,6 +100,9 @@ private void copyValue(Block from, int valueIndex, Block.Builder into) { case DOUBLE: ((DoubleBlock.Builder) into).appendDouble(((DoubleBlock) from).getDouble(valueIndex)); break; + case FLOAT: + ((FloatBlock.Builder) into).appendFloat(((FloatBlock) from).getFloat(valueIndex)); + break; default: throw new IllegalArgumentException("unknown block type " + elementType); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/PositionMergingSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/PositionMergingSourceOperator.java index 4bbd6d0af0c2a..651ec6dc191a9 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/PositionMergingSourceOperator.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/PositionMergingSourceOperator.java @@ -13,6 +13,7 @@ import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.compute.data.Page; @@ -74,6 +75,7 @@ private void copyTo(Block.Builder builder, Block in, int position, int valueCoun switch (in.elementType()) { case BOOLEAN -> ((BooleanBlock.Builder) builder).appendBoolean(((BooleanBlock) in).getBoolean(i)); case BYTES_REF -> ((BytesRefBlock.Builder) builder).appendBytesRef(((BytesRefBlock) in).getBytesRef(i, scratch)); + case FLOAT -> ((FloatBlock.Builder) builder).appendFloat(((FloatBlock) in).getFloat(i)); case DOUBLE -> ((DoubleBlock.Builder) builder).appendDouble(((DoubleBlock) in).getDouble(i)); case INT -> ((IntBlock.Builder) builder).appendInt(((IntBlock) in).getInt(i)); case LONG -> ((LongBlock.Builder) builder).appendLong(((LongBlock) in).getLong(i)); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceFloatBlockSourceOperator.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceFloatBlockSourceOperator.java new file mode 100644 index 0000000000000..db524366b381e --- /dev/null +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/SequenceFloatBlockSourceOperator.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.compute.operator; + +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Page; + +import java.util.List; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +/** + * A source operator whose output is the given float values. This operator produces pages + * containing a single Block. The Block contains the float values from the given list, in order. + */ +public class SequenceFloatBlockSourceOperator extends AbstractBlockSourceOperator { + + static final int DEFAULT_MAX_PAGE_POSITIONS = 8 * 1024; + + private final float[] values; + + public SequenceFloatBlockSourceOperator(BlockFactory blockFactory, Stream values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public SequenceFloatBlockSourceOperator(BlockFactory blockFactory, Stream values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + var l = values.toList(); + this.values = new float[l.size()]; + IntStream.range(0, l.size()).forEach(i -> this.values[i] = l.get(i)); + } + + public SequenceFloatBlockSourceOperator(BlockFactory blockFactory, List values) { + this(blockFactory, values, DEFAULT_MAX_PAGE_POSITIONS); + } + + public SequenceFloatBlockSourceOperator(BlockFactory blockFactory, List values, int maxPagePositions) { + super(blockFactory, maxPagePositions); + this.values = new float[values.size()]; + IntStream.range(0, this.values.length).forEach(i -> this.values[i] = values.get(i)); + } + + @Override + protected Page createPage(int positionOffset, int length) { + FloatVector.FixedBuilder builder = blockFactory.newFloatVectorFixedBuilder(length); + for (int i = 0; i < length; i++) { + builder.appendFloat(values[positionOffset + i]); + } + currentPosition += length; + return new Page(builder.build().asBlock()); + } + + protected int remaining() { + return values.length - currentPosition; + } +} diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java index dfa49ac134430..e535c0dddd7c2 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/mvdedupe/MultivalueDedupeTests.java @@ -57,7 +57,7 @@ public class MultivalueDedupeTests extends ESTestCase { public static List supportedTypes() { List supported = new ArrayList<>(); for (ElementType elementType : ElementType.values()) { - if (oneOf(elementType, ElementType.UNKNOWN, ElementType.DOC, ElementType.COMPOSITE)) { + if (oneOf(elementType, ElementType.UNKNOWN, ElementType.DOC, ElementType.COMPOSITE, ElementType.FLOAT)) { continue; } supported.add(elementType); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java index 40c6074fc7d3a..f01e3c18c78bc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/ExtractorTests.java @@ -43,6 +43,8 @@ public static Iterable parameters() { case COMPOSITE -> { // TODO: add later } + case FLOAT -> { + } case BYTES_REF -> { cases.add(valueTestCase("single alpha", e, TopNEncoder.UTF8, () -> randomAlphaOfLength(5))); cases.add(valueTestCase("many alpha", e, TopNEncoder.UTF8, () -> randomList(2, 10, () -> randomAlphaOfLength(5)))); diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java index b2195f205c93b..be598f100563d 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/operator/topn/TopNOperatorTests.java @@ -67,6 +67,7 @@ import static org.elasticsearch.compute.data.ElementType.BYTES_REF; import static org.elasticsearch.compute.data.ElementType.COMPOSITE; import static org.elasticsearch.compute.data.ElementType.DOUBLE; +import static org.elasticsearch.compute.data.ElementType.FLOAT; import static org.elasticsearch.compute.data.ElementType.INT; import static org.elasticsearch.compute.data.ElementType.LONG; import static org.elasticsearch.compute.operator.topn.TopNEncoder.DEFAULT_SORTABLE; @@ -330,6 +331,21 @@ public void testCompareLongs() { ); } + public void testCompareFloats() { + BlockFactory blockFactory = blockFactory(); + testCompare( + new Page( + blockFactory.newFloatBlockBuilder(2).appendFloat(-Float.MAX_VALUE).appendFloat(randomFloatBetween(-1000, -1, true)).build(), + blockFactory.newFloatBlockBuilder(2).appendFloat(randomFloatBetween(-1000, -1, true)).appendFloat(0.0f).build(), + blockFactory.newFloatBlockBuilder(2).appendFloat(0).appendFloat(randomFloatBetween(1, 1000, true)).build(), + blockFactory.newFloatBlockBuilder(2).appendFloat(randomLongBetween(1, 1000)).appendFloat(Float.MAX_VALUE).build(), + blockFactory.newFloatBlockBuilder(2).appendFloat(0.0f).appendFloat(Float.MAX_VALUE).build() + ), + FLOAT, + DEFAULT_SORTABLE + ); + } + public void testCompareDoubles() { BlockFactory blockFactory = blockFactory(); testCompare( diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index 7f0d9b9170c5e..d7e146cd6d7c1 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -43,6 +43,7 @@ public class EsqlSecurityIT extends ESRestTestCase { .distribution(DistributionType.DEFAULT) .setting("xpack.license.self_generated.type", "trial") .setting("xpack.security.enabled", "true") + .setting("xpack.ml.enabled", "false") .rolesFile(Resource.fromClasspath("roles.yml")) .user("test-admin", "x-pack-test-password", "test-admin", true) .user("user1", "x-pack-test-password", "user1", false) @@ -51,6 +52,7 @@ public class EsqlSecurityIT extends ESRestTestCase { .user("user4", "x-pack-test-password", "user4", false) .user("user5", "x-pack-test-password", "user5", false) .user("fls_user", "x-pack-test-password", "fls_user", false) + .user("metadata1_read2", "x-pack-test-password", "metadata1_read2", false) .build(); @Override @@ -135,6 +137,21 @@ public void testUnauthorizedIndices() throws IOException { assertThat(error.getResponse().getStatusLine().getStatusCode(), equalTo(400)); } + public void testInsufficientPrivilege() { + Exception error = expectThrows( + Exception.class, + () -> runESQLCommand("metadata1_read2", "FROM index-user1,index-user2 | STATS sum=sum(value)") + ); + assertThat( + error.getMessage(), + containsString( + "unauthorized for user [test-admin] run as [metadata1_read2] " + + "with effective roles [metadata1_read2] on indices [index-user1], " + + "this action is granted by the index privileges [read,all]" + ) + ); + } + public void testDocumentLevelSecurity() throws Exception { Response resp = runESQLCommand("user3", "from index | stats sum=sum(value)"); assertOK(resp); diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml index 7d134103afd28..6225711918608 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml @@ -32,6 +32,14 @@ user2: - create_index - indices:admin/refresh +metadata1_read2: + cluster: [] + indices: + - names: [ 'index-user1' ] + privileges: [ 'view_index_metadata' ] + - names: [ 'index-user2' ] + privileges: [ 'read' ] + user3: cluster: [] indices: diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 6cf6fe3984b32..807d6cff1966c 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -97,7 +97,6 @@ protected void shouldSkipTest(String testName) throws IOException { super.shouldSkipTest(testName); checkCapabilities(remoteClusterClient(), remoteFeaturesService(), testName, testCase); assumeFalse("can't test with _index metadata", hasIndexMetadata(testCase.query)); - assumeTrue("can't test with metrics across cluster", hasMetricsCommand(testCase.query)); assumeTrue("Test " + testName + " is skipped on " + Clusters.oldVersion(), isEnabled(testName, Clusters.oldVersion())); } @@ -195,7 +194,6 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas String query = testCase.query; String[] commands = query.split("\\|"); String first = commands[0].trim(); - if (commands[0].toLowerCase(Locale.ROOT).startsWith("from")) { String[] parts = commands[0].split("(?i)metadata"); assert parts.length >= 1 : parts; @@ -208,6 +206,14 @@ static CsvSpecReader.CsvTestCase convertToRemoteIndices(CsvSpecReader.CsvTestCas var newFrom = "FROM " + remoteIndices + " " + commands[0].substring(fromStatement.length()); testCase.query = newFrom + query.substring(first.length()); } + if (commands[0].toLowerCase(Locale.ROOT).startsWith("metrics")) { + String[] parts = commands[0].split("\\s+"); + assert parts.length >= 2 : commands[0]; + String[] indices = parts[1].split(","); + parts[1] = Arrays.stream(indices).map(index -> "*:" + index + "," + index).collect(Collectors.joining(",")); + String newNewMetrics = String.join(" ", parts); + testCase.query = newNewMetrics + query.substring(first.length()); + } int offset = testCase.query.length() - query.length(); if (offset != 0) { final String pattern = "Line (\\d+):(\\d+):"; @@ -236,8 +242,4 @@ static boolean hasIndexMetadata(String query) { } return false; } - - static boolean hasMetricsCommand(String query) { - return Arrays.stream(query.split("\\|")).anyMatch(s -> s.trim().toLowerCase(Locale.ROOT).startsWith("metrics")); - } } diff --git a/x-pack/plugin/esql/qa/testFixtures/build.gradle b/x-pack/plugin/esql/qa/testFixtures/build.gradle index 520873a6cb03e..e8a95011100f5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/build.gradle +++ b/x-pack/plugin/esql/qa/testFixtures/build.gradle @@ -1,9 +1,9 @@ apply plugin: 'elasticsearch.java' - +apply plugin: org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin dependencies { implementation project(':x-pack:plugin:esql:compute') - compileOnly project(':x-pack:plugin:esql') + implementation project(':x-pack:plugin:esql') compileOnly project(path: xpackModule('core')) implementation project(":libs:elasticsearch-x-content") implementation project(':client:rest') @@ -11,7 +11,14 @@ dependencies { implementation project(':test:framework') api(testArtifact(project(xpackModule('esql-core')))) implementation project(':server') - api "net.sf.supercsv:super-csv:${versions.supercsv}" + implementation "net.sf.supercsv:super-csv:${versions.supercsv}" +} + +/** + * This is needed for CsvTestsDataLoaderTests to reflect the classpath that CsvTestsDataLoader actually uses when "main" method is executed. + */ +tasks.named("test").configure { + classpath = classpath - (configurations.resolveableCompileOnly - configurations.runtimeClasspath) } /** diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java index af3af033efd4c..875058ba6e0e4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java @@ -41,7 +41,6 @@ import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; public final class CsvAssert { @@ -110,6 +109,9 @@ private static void assertMetadata( if (actualType == Type.INTEGER && expectedType == Type.LONG) { actualType = Type.LONG; } + if (actualType == null) { + actualType = Type.NULL; + } assertEquals( "Different column type for column [" + expectedName + "] (" + expectedType + " != " + actualType + ")", @@ -188,7 +190,13 @@ public static void assertData( for (int row = 0; row < expectedValues.size(); row++) { try { - assertTrue("Expected more data but no more entries found after [" + row + "]", row < actualValues.size()); + if (row >= actualValues.size()) { + if (dataFailures.isEmpty()) { + fail("Expected more data but no more entries found after [" + row + "]"); + } else { + dataFailure(dataFailures, "Expected more data but no more entries found after [" + row + "]\n"); + } + } if (logger != null) { logger.info(row(actualValues, row)); @@ -257,7 +265,11 @@ public static void assertData( } private static void dataFailure(List dataFailures) { - fail("Data mismatch:\n" + dataFailures.stream().map(f -> { + dataFailure(dataFailures, ""); + } + + private static void dataFailure(List dataFailures, String prefixError) { + fail(prefixError + "Data mismatch:\n" + dataFailures.stream().map(f -> { Description description = new StringDescription(); ListMatcher expected; if (f.expected instanceof List e) { diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index c27d0a6fbb865..ad7c3fba1683e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -499,6 +499,7 @@ public static Type asType(ElementType elementType, Type actualType) { return switch (elementType) { case INT -> INTEGER; case LONG -> LONG; + case FLOAT -> FLOAT; case DOUBLE -> DOUBLE; case NULL -> NULL; case BYTES_REF -> bytesRefBlockType(actualType); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 1c1ec3194fef5..ec5770e8ce70b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -57,6 +57,16 @@ public class CsvTestsDataLoader { private static final TestsDataset LANGUAGES = new TestsDataset("languages", "mapping-languages.json", "languages.csv"); private static final TestsDataset UL_LOGS = new TestsDataset("ul_logs", "mapping-ul_logs.json", "ul_logs.csv"); private static final TestsDataset SAMPLE_DATA = new TestsDataset("sample_data", "mapping-sample_data.json", "sample_data.csv"); + private static final TestsDataset SAMPLE_DATA_STR = new TestsDataset( + "sample_data_str", + "mapping-sample_data_str.json", + "sample_data_str.csv" + ); + private static final TestsDataset SAMPLE_DATA_TS_LONG = new TestsDataset( + "sample_data_ts_long", + "mapping-sample_data_ts_long.json", + "sample_data_ts_long.csv" + ); private static final TestsDataset CLIENT_IPS = new TestsDataset("clientips", "mapping-clientips.json", "clientips.csv"); private static final TestsDataset CLIENT_CIDR = new TestsDataset("client_cidr", "mapping-client_cidr.json", "client_cidr.csv"); private static final TestsDataset AGES = new TestsDataset("ages", "mapping-ages.json", "ages.csv"); @@ -95,6 +105,8 @@ public class CsvTestsDataLoader { Map.entry(LANGUAGES.indexName, LANGUAGES), Map.entry(UL_LOGS.indexName, UL_LOGS), Map.entry(SAMPLE_DATA.indexName, SAMPLE_DATA), + Map.entry(SAMPLE_DATA_STR.indexName, SAMPLE_DATA_STR), + Map.entry(SAMPLE_DATA_TS_LONG.indexName, SAMPLE_DATA_TS_LONG), Map.entry(CLIENT_IPS.indexName, CLIENT_IPS), Map.entry(CLIENT_CIDR.indexName, CLIENT_CIDR), Map.entry(AGES.indexName, AGES), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec index 377d6d6678032..35e1101becbf9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup.csv-spec @@ -1,5 +1,5 @@ keywordByInt -required_capability: lookup_command +required_capability: tables_types FROM employees | SORT emp_no | LIMIT 4 @@ -17,7 +17,7 @@ emp_no:integer | languages:integer | lang_name:keyword ; keywordByMvInt -required_capability: lookup_command +required_capability: tables_types ROW int=[1, 2, 3] | LOOKUP int_number_names ON int ; @@ -27,7 +27,7 @@ int:integer | name:keyword ; keywordByDupeInt -required_capability: lookup_command +required_capability: tables_types ROW int=[1, 1, 1] | LOOKUP int_number_names ON int ; @@ -37,7 +37,7 @@ int:integer | name:keyword ; intByKeyword -required_capability: lookup_command +required_capability: tables_types ROW name="two" | LOOKUP int_number_names ON name ; @@ -48,7 +48,7 @@ name:keyword | int:integer keywordByLong -required_capability: lookup_command +required_capability: tables_types FROM employees | SORT emp_no | LIMIT 4 @@ -66,7 +66,7 @@ emp_no:integer | languages:long | lang_name:keyword ; longByKeyword -required_capability: lookup_command +required_capability: tables_types ROW name="two" | LOOKUP long_number_names ON name ; @@ -76,7 +76,7 @@ name:keyword | long:long ; keywordByFloat -required_capability: lookup_command +required_capability: tables_types FROM employees | SORT emp_no | LIMIT 4 @@ -94,7 +94,7 @@ emp_no:integer | height:double | height_name:keyword ; floatByKeyword -required_capability: lookup_command +required_capability: tables_types ROW name="two point zero eight" | LOOKUP double_number_names ON name ; @@ -104,7 +104,7 @@ two point zero eight | 2.08 ; floatByNullMissing -required_capability: lookup_command +required_capability: tables_types ROW name=null | LOOKUP double_number_names ON name ; @@ -114,7 +114,7 @@ name:null | double:double ; floatByNullMatching -required_capability: lookup_command +required_capability: tables_types ROW name=null | LOOKUP double_number_names_with_null ON name ; @@ -124,7 +124,7 @@ name:null | double:double ; intIntByKeywordKeyword -required_capability: lookup_command +required_capability: tables_types ROW aa="foo", ab="zoo" | LOOKUP big ON aa, ab ; @@ -134,7 +134,7 @@ foo | zoo | 1 | -1 ; intIntByKeywordKeywordMissing -required_capability: lookup_command +required_capability: tables_types ROW aa="foo", ab="zoi" | LOOKUP big ON aa, ab ; @@ -144,7 +144,7 @@ foo | zoi | null | null ; intIntByKeywordKeywordSameValues -required_capability: lookup_command +required_capability: tables_types ROW aa="foo", ab="foo" | LOOKUP big ON aa, ab ; @@ -154,7 +154,7 @@ foo | foo | 2 | -2 ; intIntByKeywordKeywordSameValuesMissing -required_capability: lookup_command +required_capability: tables_types ROW aa="bar", ab="bar" | LOOKUP big ON aa, ab ; @@ -164,7 +164,7 @@ bar | bar | null | null ; lookupBeforeStats -required_capability: lookup_command +required_capability: tables_types FROM employees | RENAME languages AS int | LOOKUP int_number_names ON int @@ -182,7 +182,7 @@ height:double | languages:keyword ; lookupAfterStats -required_capability: lookup_command +required_capability: tables_types FROM employees | STATS int=TO_INT(AVG(height)) | LOOKUP int_number_names ON int @@ -194,7 +194,7 @@ two // Makes sure the LOOKUP squashes previous names doesNotDuplicateNames -required_capability: lookup_command +required_capability: tables_types FROM employees | SORT emp_no | LIMIT 4 @@ -213,7 +213,7 @@ emp_no:integer | languages:long | name:keyword ; lookupBeforeSort -required_capability: lookup_command +required_capability: tables_types FROM employees | WHERE emp_no < 10005 | RENAME languages AS int @@ -231,7 +231,7 @@ languages:keyword | emp_no:integer ; lookupAfterSort -required_capability: lookup_command +required_capability: tables_types FROM employees | WHERE emp_no < 10005 | SORT languages ASC, emp_no ASC @@ -253,7 +253,7 @@ languages:keyword | emp_no:integer // named "lookup" // rowNamedLookup -required_capability: lookup_command +required_capability: tables_types ROW lookup = "a" ; @@ -262,7 +262,7 @@ lookup:keyword ; rowNamedLOOKUP -required_capability: lookup_command +required_capability: tables_types ROW LOOKUP = "a" ; @@ -271,7 +271,7 @@ LOOKUP:keyword ; evalNamedLookup -required_capability: lookup_command +required_capability: tables_types ROW a = "a" | EVAL lookup = CONCAT(a, "1") ; @@ -280,7 +280,7 @@ a:keyword | lookup:keyword ; dissectNamedLookup -required_capability: lookup_command +required_capability: tables_types row a = "foo bar" | dissect a "foo %{lookup}"; a:keyword | lookup:keyword @@ -288,7 +288,7 @@ a:keyword | lookup:keyword ; renameIntoLookup -required_capability: lookup_command +required_capability: tables_types row a = "foo bar" | RENAME a AS lookup; lookup:keyword @@ -296,7 +296,7 @@ lookup:keyword ; sortOnLookup -required_capability: lookup_command +required_capability: tables_types ROW lookup = "a" | SORT lookup ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-sample_data_str.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-sample_data_str.json new file mode 100644 index 0000000000000..9e97de8c92928 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-sample_data_str.json @@ -0,0 +1,16 @@ +{ + "properties": { + "@timestamp": { + "type": "date" + }, + "client_ip": { + "type": "keyword" + }, + "event_duration": { + "type": "long" + }, + "message": { + "type": "keyword" + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-sample_data_ts_long.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-sample_data_ts_long.json new file mode 100644 index 0000000000000..ecf21a2a919d0 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-sample_data_ts_long.json @@ -0,0 +1,16 @@ +{ + "properties": { + "@timestamp": { + "type": "long" + }, + "client_ip": { + "type": "ip" + }, + "event_duration": { + "type": "long" + }, + "message": { + "type": "keyword" + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec index 2cdd5c1dfd931..0fb35b4253d6d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec @@ -38,10 +38,10 @@ double e() "double log(?base:integer|unsigned_long|long|double, number:integer|unsigned_long|long|double)" "double log10(number:double|integer|long|unsigned_long)" "keyword|text ltrim(string:keyword|text)" -"double|integer|long max(number:double|integer|long)" +"double|integer|long|date max(number:double|integer|long|date)" "double|integer|long median(number:double|integer|long)" "double|integer|long median_absolute_deviation(number:double|integer|long)" -"double|integer|long min(number:double|integer|long)" +"double|integer|long|date min(number:double|integer|long|date)" "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version mv_append(field1:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version, field2:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version)" "double mv_avg(number:double|integer|long|unsigned_long)" "keyword mv_concat(string:text|keyword, delim:text|keyword)" @@ -109,6 +109,7 @@ double tau() "keyword|text to_upper(str:keyword|text)" "version to_ver(field:keyword|text|version)" "version to_version(field:keyword|text|version)" +"double|integer|long|date top_list(field:double|integer|long|date, limit:integer, order:keyword)" "keyword|text trim(string:keyword|text)" "boolean|date|double|integer|ip|keyword|long|text|version values(field:boolean|date|double|integer|ip|keyword|long|text|version)" ; @@ -155,10 +156,10 @@ locate |[string, substring, start] |["keyword|text", "keyword|te log |[base, number] |["integer|unsigned_long|long|double", "integer|unsigned_long|long|double"] |["Base of logarithm. If `null`\, the function returns `null`. If not provided\, this function returns the natural logarithm (base e) of a value.", "Numeric expression. If `null`\, the function returns `null`."] log10 |number |"double|integer|long|unsigned_long" |Numeric expression. If `null`, the function returns `null`. ltrim |string |"keyword|text" |String expression. If `null`, the function returns `null`. -max |number |"double|integer|long" |[""] +max |number |"double|integer|long|date" |[""] median |number |"double|integer|long" |[""] median_absolut|number |"double|integer|long" |[""] -min |number |"double|integer|long" |[""] +min |number |"double|integer|long|date" |[""] mv_append |[field1, field2] |["boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version", "boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version"] | ["", ""] mv_avg |number |"double|integer|long|unsigned_long" |Multivalue expression. mv_concat |[string, delim] |["text|keyword", "text|keyword"] |[Multivalue expression., Delimiter.] @@ -226,6 +227,7 @@ to_unsigned_lo|field |"boolean|date|keyword|text|d to_upper |str |"keyword|text" |String expression. If `null`, the function returns `null`. to_ver |field |"keyword|text|version" |Input value. The input can be a single- or multi-valued column or an expression. to_version |field |"keyword|text|version" |Input value. The input can be a single- or multi-valued column or an expression. +top_list |[field, limit, order] |["double|integer|long|date", integer, keyword] |[The field to collect the top values for.,The maximum number of values to collect.,The order to calculate the top values. Either `asc` or `desc`.] trim |string |"keyword|text" |String expression. If `null`, the function returns `null`. values |field |"boolean|date|double|integer|ip|keyword|long|text|version" |[""] ; @@ -344,6 +346,7 @@ to_unsigned_lo|Converts an input value to an unsigned long value. If the input p to_upper |Returns a new string representing the input string converted to upper case. to_ver |Converts an input string to a version value. to_version |Converts an input string to a version value. +top_list |Collects the top values for a field. Includes repeated values. trim |Removes leading and trailing whitespaces from a string. values |Collect values for a field. ; @@ -392,10 +395,10 @@ locate |integer log |double |[true, false] |false |false log10 |double |false |false |false ltrim |"keyword|text" |false |false |false -max |"double|integer|long" |false |false |true +max |"double|integer|long|date" |false |false |true median |"double|integer|long" |false |false |true median_absolut|"double|integer|long" |false |false |true -min |"double|integer|long" |false |false |true +min |"double|integer|long|date" |false |false |true mv_append |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|version" |[false, false] |false |false mv_avg |double |false |false |false mv_concat |keyword |[false, false] |false |false @@ -463,6 +466,7 @@ to_unsigned_lo|unsigned_long to_upper |"keyword|text" |false |false |false to_ver |version |false |false |false to_version |version |false |false |false +top_list |"double|integer|long|date" |[false, false, false] |false |true trim |"keyword|text" |false |false |false values |"boolean|date|double|integer|ip|keyword|long|text|version" |false |false |true ; @@ -483,5 +487,5 @@ countFunctions#[skip:-8.14.99, reason:BIN added] meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c; a:long | b:long | c:long -109 | 109 | 109 +110 | 110 | 110 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_str.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_str.csv new file mode 100644 index 0000000000000..bc98671adc7ff --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_str.csv @@ -0,0 +1,8 @@ +@timestamp:date,client_ip:keyword,event_duration:long,message:keyword +2023-10-23T13:55:01.543Z,172.21.3.15,1756467,Connected to 10.1.0.1 +2023-10-23T13:53:55.832Z,172.21.3.15,5033755,Connection error +2023-10-23T13:52:55.015Z,172.21.3.15,8268153,Connection error +2023-10-23T13:51:54.732Z,172.21.3.15,725448,Connection error +2023-10-23T13:33:34.937Z,172.21.0.5,1232382,Disconnected +2023-10-23T12:27:28.948Z,172.21.2.113,2764889,Connected to 10.1.0.2 +2023-10-23T12:15:03.360Z,172.21.2.162,3450233,Connected to 10.1.0.3 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_long.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_long.csv new file mode 100644 index 0000000000000..2a6add2ea624d --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/sample_data_ts_long.csv @@ -0,0 +1,8 @@ +@timestamp:long,client_ip:ip,event_duration:long,message:keyword +1698069301543,172.21.3.15,1756467,Connected to 10.1.0.1 +1698069235832,172.21.3.15,5033755,Connection error +1698069175015,172.21.3.15,8268153,Connection error +1698069114732,172.21.3.15,725448,Connection error +1698068014937,172.21.0.5,1232382,Disconnected +1698064048948,172.21.2.113,2764889,Connected to 10.1.0.2 +1698063303360,172.21.2.162,3450233,Connected to 10.1.0.3 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top_list.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top_list.csv-spec new file mode 100644 index 0000000000000..c24f6a7e70954 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats_top_list.csv-spec @@ -0,0 +1,156 @@ +topList +required_capability: agg_top_list +// tag::top-list[] +FROM employees +| STATS top_salaries = TOP_LIST(salary, 3, "desc"), top_salary = MAX(salary) +// end::top-list[] +; + +// tag::top-list-result[] +top_salaries:integer | top_salary:integer +[74999, 74970, 74572] | 74999 +// end::top-list-result[] +; + +topListAllTypesAsc +required_capability: agg_top_list +FROM employees +| STATS + date = TOP_LIST(hire_date, 2, "asc"), + double = TOP_LIST(salary_change, 2, "asc"), + integer = TOP_LIST(salary, 2, "asc"), + long = TOP_LIST(salary_change.long, 2, "asc") +; + +date:date | double:double | integer:integer | long:long +[1985-02-18T00:00:00.000Z,1985-02-24T00:00:00.000Z] | [-9.81,-9.28] | [25324,25945] | [-9,-9] +; + +topListAllTypesDesc +required_capability: agg_top_list +FROM employees +| STATS + date = TOP_LIST(hire_date, 2, "desc"), + double = TOP_LIST(salary_change, 2, "desc"), + integer = TOP_LIST(salary, 2, "desc"), + long = TOP_LIST(salary_change.long, 2, "desc") +; + +date:date | double:double | integer:integer | long:long +[1999-04-30T00:00:00.000Z,1997-05-19T00:00:00.000Z] | [14.74,14.68] | [74999,74970] | [14,14] +; + +topListAllTypesRow +required_capability: agg_top_list +ROW + constant_date=TO_DATETIME("1985-02-18T00:00:00.000Z"), + constant_double=-9.81, + constant_integer=25324, + constant_long=TO_LONG(-9) +| STATS + date = TOP_LIST(constant_date, 2, "asc"), + double = TOP_LIST(constant_double, 2, "asc"), + integer = TOP_LIST(constant_integer, 2, "asc"), + long = TOP_LIST(constant_long, 2, "asc") +| keep date, double, integer, long +; + +date:date | double:double | integer:integer | long:long +1985-02-18T00:00:00.000Z | -9.81 | 25324 | -9 +; + +topListSomeBuckets +required_capability: agg_top_list +FROM employees +| STATS top_salary = TOP_LIST(salary, 2, "desc") by still_hired +| sort still_hired asc +; + +top_salary:integer | still_hired:boolean +[74999,74970] | false +[74572,73578] | true +; + +topListManyBuckets +required_capability: agg_top_list +FROM employees +| STATS top_salary = TOP_LIST(salary, 2, "desc") by x=emp_no, y=emp_no+1 +| sort x asc +| limit 3 +; + +top_salary:integer | x:integer | y:integer +57305 | 10001 | 10002 +56371 | 10002 | 10003 +61805 | 10003 | 10004 +; + +topListMultipleStats +required_capability: agg_top_list +FROM employees +| STATS top_salary = TOP_LIST(salary, 1, "desc") by emp_no +| STATS top_salary = TOP_LIST(top_salary, 3, "asc") +; + +top_salary:integer +[25324,25945,25976] +; + +topListAllTypesMin +required_capability: agg_top_list +FROM employees +| STATS + date = TOP_LIST(hire_date, 1, "asc"), + double = TOP_LIST(salary_change, 1, "asc"), + integer = TOP_LIST(salary, 1, "asc"), + long = TOP_LIST(salary_change.long, 1, "asc") +; + +date:date | double:double | integer:integer | long:long +1985-02-18T00:00:00.000Z | -9.81 | 25324 | -9 +; + +topListAllTypesMax +required_capability: agg_top_list +FROM employees +| STATS + date = TOP_LIST(hire_date, 1, "desc"), + double = TOP_LIST(salary_change, 1, "desc"), + integer = TOP_LIST(salary, 1, "desc"), + long = TOP_LIST(salary_change.long, 1, "desc") +; + +date:date | double:double | integer:integer | long:long +1999-04-30T00:00:00.000Z | 14.74 | 74999 | 14 +; + +topListAscDesc +required_capability: agg_top_list +FROM employees +| STATS top_asc = TOP_LIST(salary, 3, "asc"), top_desc = TOP_LIST(salary, 3, "desc") +; + +top_asc:integer | top_desc:integer +[25324, 25945, 25976] | [74999, 74970, 74572] +; + +topListEmpty +required_capability: agg_top_list +FROM employees +| WHERE salary < 0 +| STATS top = TOP_LIST(salary, 3, "asc") +; + +top:integer +null +; + +topListDuplicates +required_capability: agg_top_list +FROM employees +| STATS integer = TOP_LIST(languages, 2, "desc") +; + +integer:integer +[5, 5] +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec new file mode 100644 index 0000000000000..ee8c4be385e0f --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -0,0 +1,719 @@ +singleIndexIp +FROM sample_data +| EVAL client_ip = TO_IP(client_ip) +| KEEP @timestamp, client_ip, event_duration, message +| SORT @timestamp DESC +; + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +singleIndexWhereIpLike +FROM sample_data +| WHERE TO_STRING(client_ip) LIKE "172.21.2.*" +| KEEP @timestamp, event_duration, message +| SORT @timestamp DESC +; + +@timestamp:date | event_duration:long | message:keyword +2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 +2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 +; + +singleIndexTsLong +FROM sample_data_ts_long +| EVAL @timestamp = TO_DATETIME(@timestamp) +| KEEP @timestamp, client_ip, event_duration, message +| SORT @timestamp DESC +; + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +singleIndexIpStats +FROM sample_data +| EVAL client_ip = TO_IP(client_ip) +| STATS count=count(*) BY client_ip +| SORT count DESC, client_ip ASC +| KEEP count, client_ip +; + +count:long | client_ip:ip +4 | 172.21.3.15 +1 | 172.21.0.5 +1 | 172.21.2.113 +1 | 172.21.2.162 +; + +singleIndexIpStringStats +FROM sample_data_str +| EVAL client_ip = TO_IP(client_ip) +| STATS count=count(*) BY client_ip +| SORT count DESC, client_ip ASC +| KEEP count, client_ip +; + +count:long | client_ip:ip +4 | 172.21.3.15 +1 | 172.21.0.5 +1 | 172.21.2.113 +1 | 172.21.2.162 +; + +multiIndexIpString +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_str METADATA _index +| EVAL client_ip = TO_IP(client_ip) +| KEEP _index, @timestamp, client_ip, event_duration, message +| SORT _index ASC, @timestamp DESC +; + +_index:keyword | @timestamp:date | client_ip:ip | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_str | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_str | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_str | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_str | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_str | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_str | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexIpStringRename +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_str METADATA _index +| EVAL host_ip = TO_IP(client_ip) +| KEEP _index, @timestamp, host_ip, event_duration, message +| SORT _index ASC, @timestamp DESC +; + +_index:keyword | @timestamp:date | host_ip:ip | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_str | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_str | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_str | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_str | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_str | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_str | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexIpStringRenameToString +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_str METADATA _index +| EVAL host_ip = TO_STRING(TO_IP(client_ip)) +| KEEP _index, @timestamp, host_ip, event_duration, message +| SORT _index ASC, @timestamp DESC +; + +_index:keyword | @timestamp:date | host_ip:keyword | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_str | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_str | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_str | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_str | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_str | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_str | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexWhereIpString +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_str METADATA _index +| WHERE STARTS_WITH(TO_STRING(client_ip), "172.21.2") +| KEEP _index, @timestamp, event_duration, message +| SORT _index ASC, @timestamp DESC +; + +_index:keyword | @timestamp:date | event_duration:long | message:keyword +sample_data | 2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 +sample_data_str | 2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 +sample_data_str | 2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 +; + +multiIndexWhereIpStringLike +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_str METADATA _index +| WHERE TO_STRING(client_ip) LIKE "172.21.2.*" +| KEEP _index, @timestamp, event_duration, message +| SORT _index ASC, @timestamp DESC +; + +_index:keyword | @timestamp:date | event_duration:long | message:keyword +sample_data | 2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 +sample_data_str | 2023-10-23T12:27:28.948Z | 2764889 | Connected to 10.1.0.2 +sample_data_str | 2023-10-23T12:15:03.360Z | 3450233 | Connected to 10.1.0.3 +; + +multiIndexIpStringStats +required_capability: union_types + +FROM sample_data, sample_data_str +| EVAL client_ip = TO_IP(client_ip) +| STATS count=count(*) BY client_ip +| SORT count DESC, client_ip ASC +| KEEP count, client_ip +; + +count:long | client_ip:ip +8 | 172.21.3.15 +2 | 172.21.0.5 +2 | 172.21.2.113 +2 | 172.21.2.162 +; + +multiIndexIpStringRenameStats +required_capability: union_types + +FROM sample_data, sample_data_str +| EVAL host_ip = TO_IP(client_ip) +| STATS count=count(*) BY host_ip +| SORT count DESC, host_ip ASC +| KEEP count, host_ip +; + +count:long | host_ip:ip +8 | 172.21.3.15 +2 | 172.21.0.5 +2 | 172.21.2.113 +2 | 172.21.2.162 +; + +multiIndexIpStringRenameToStringStats +required_capability: union_types + +FROM sample_data, sample_data_str +| EVAL host_ip = TO_STRING(TO_IP(client_ip)) +| STATS count=count(*) BY host_ip +| SORT count DESC, host_ip ASC +| KEEP count, host_ip +; + +count:long | host_ip:keyword +8 | 172.21.3.15 +2 | 172.21.0.5 +2 | 172.21.2.113 +2 | 172.21.2.162 +; + +multiIndexIpStringStatsInline +required_capability: union_types +required_capability: union_types_inline_fix + +FROM sample_data, sample_data_str +| STATS count=count(*) BY client_ip = TO_IP(client_ip) +| SORT count DESC, client_ip ASC +| KEEP count, client_ip +; + +count:long | client_ip:ip +8 | 172.21.3.15 +2 | 172.21.0.5 +2 | 172.21.2.113 +2 | 172.21.2.162 +; + +multiIndexWhereIpStringStats +required_capability: union_types + +FROM sample_data, sample_data_str +| WHERE STARTS_WITH(TO_STRING(client_ip), "172.21.2") +| STATS count=count(*) BY message +| SORT count DESC, message ASC +| KEEP count, message +; + +count:long | message:keyword +2 | Connected to 10.1.0.2 +2 | Connected to 10.1.0.3 +; + +multiIndexTsLong +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_ts_long METADATA _index +| EVAL @timestamp = TO_DATETIME(@timestamp) +| KEEP _index, @timestamp, client_ip, event_duration, message +| SORT _index ASC, @timestamp DESC +; + +_index:keyword | @timestamp:date | client_ip:ip | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_ts_long | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_ts_long | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_ts_long | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_ts_long | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_ts_long | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexTsLongRename +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_ts_long METADATA _index +| EVAL ts = TO_DATETIME(@timestamp) +| KEEP _index, ts, client_ip, event_duration, message +| SORT _index ASC, ts DESC +; + +_index:keyword | ts:date | client_ip:ip | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_ts_long | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_ts_long | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_ts_long | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_ts_long | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_ts_long | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexTsLongRenameToString +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_ts_long METADATA _index +| EVAL ts = TO_STRING(TO_DATETIME(@timestamp)) +| KEEP _index, ts, client_ip, event_duration, message +| SORT _index ASC, ts DESC +; + +_index:keyword | ts:keyword | client_ip:ip | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_ts_long | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_ts_long | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_ts_long | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_ts_long | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_ts_long | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexWhereTsLong +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data, sample_data_ts_long METADATA _index +| WHERE TO_LONG(@timestamp) < 1698068014937 +| KEEP _index, client_ip, event_duration, message +| SORT _index ASC, client_ip ASC +; + +_index:keyword | client_ip:ip | event_duration:long | message:keyword +sample_data | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexTsLongStats +required_capability: union_types + +FROM sample_data, sample_data_ts_long +| EVAL @timestamp = DATE_TRUNC(1 hour, TO_DATETIME(@timestamp)) +| STATS count=count(*) BY @timestamp +| SORT count DESC, @timestamp ASC +| KEEP count, @timestamp +; + +count:long | @timestamp:date +10 | 2023-10-23T13:00:00.000Z +4 | 2023-10-23T12:00:00.000Z +; + +multiIndexTsLongRenameStats +required_capability: union_types + +FROM sample_data, sample_data_ts_long +| EVAL hour = DATE_TRUNC(1 hour, TO_DATETIME(@timestamp)) +| STATS count=count(*) BY hour +| SORT count DESC, hour ASC +| KEEP count, hour +; + +count:long | hour:date +10 | 2023-10-23T13:00:00.000Z +4 | 2023-10-23T12:00:00.000Z +; + +multiIndexTsLongRenameToDatetimeToStringStats +required_capability: union_types + +FROM sample_data, sample_data_ts_long +| EVAL hour = LEFT(TO_STRING(TO_DATETIME(@timestamp)), 13) +| STATS count=count(*) BY hour +| SORT count DESC, hour ASC +| KEEP count, hour +; + +count:long | hour:keyword +10 | 2023-10-23T13 +4 | 2023-10-23T12 +; + +multiIndexTsLongRenameToStringStats +required_capability: union_types + +FROM sample_data, sample_data_ts_long +| EVAL mess = LEFT(TO_STRING(@timestamp), 7) +| STATS count=count(*) BY mess +| SORT count DESC, mess DESC +| KEEP count, mess +; + +count:long | mess:keyword +7 | 2023-10 +4 | 1698069 +1 | 1698068 +1 | 1698064 +1 | 1698063 +; + +multiIndexTsLongStatsInline +required_capability: union_types + +FROM sample_data, sample_data_ts_long +| STATS count=COUNT(*), max=MAX(TO_DATETIME(@timestamp)) +| KEEP count, max +; + +count:long | max:date +14 | 2023-10-23T13:55:01.543Z +; + +multiIndexTsLongStatsInlineDropped +required_capability: union_types + +FROM sample_data, sample_data_ts_long +| STATS count=COUNT(*), max=MAX(TO_DATETIME(@timestamp)) +| KEEP count +; + +count:long +14 +; + +multiIndexWhereTsLongStats +required_capability: union_types + +FROM sample_data, sample_data_ts_long +| WHERE TO_LONG(@timestamp) < 1698068014937 +| STATS count=count(*) BY message +| SORT count DESC, message ASC +| KEEP count, message +; + +count:long | message:keyword +2 | Connected to 10.1.0.2 +2 | Connected to 10.1.0.3 +; + +multiIndexIpStringTsLong +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data* METADATA _index +| EVAL @timestamp = TO_DATETIME(@timestamp), client_ip = TO_IP(client_ip) +| KEEP _index, @timestamp, client_ip, event_duration, message +| SORT _index ASC, @timestamp DESC +; + +_index:keyword | @timestamp:date | client_ip:ip | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_str | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_str | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_str | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_str | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_str | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_str | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_ts_long | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_ts_long | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_ts_long | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_ts_long | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_ts_long | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexIpStringTsLongDropped +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data* METADATA _index +| EVAL @timestamp = TO_DATETIME(@timestamp), client_ip = TO_IP(client_ip) +| KEEP _index, event_duration, message +| SORT _index ASC, event_duration ASC +; + +_index:keyword | event_duration:long | message:keyword +sample_data | 725448 | Connection error +sample_data | 1232382 | Disconnected +sample_data | 1756467 | Connected to 10.1.0.1 +sample_data | 2764889 | Connected to 10.1.0.2 +sample_data | 3450233 | Connected to 10.1.0.3 +sample_data | 5033755 | Connection error +sample_data | 8268153 | Connection error +sample_data_str | 725448 | Connection error +sample_data_str | 1232382 | Disconnected +sample_data_str | 1756467 | Connected to 10.1.0.1 +sample_data_str | 2764889 | Connected to 10.1.0.2 +sample_data_str | 3450233 | Connected to 10.1.0.3 +sample_data_str | 5033755 | Connection error +sample_data_str | 8268153 | Connection error +sample_data_ts_long | 725448 | Connection error +sample_data_ts_long | 1232382 | Disconnected +sample_data_ts_long | 1756467 | Connected to 10.1.0.1 +sample_data_ts_long | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 5033755 | Connection error +sample_data_ts_long | 8268153 | Connection error +; + +multiIndexIpStringTsLongRename +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data* METADATA _index +| EVAL ts = TO_DATETIME(@timestamp), host_ip = TO_IP(client_ip) +| KEEP _index, ts, host_ip, event_duration, message +| SORT _index ASC, ts DESC +; + +_index:keyword | ts:date | host_ip:ip | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_str | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_str | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_str | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_str | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_str | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_str | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_ts_long | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_ts_long | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_ts_long | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_ts_long | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_ts_long | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexIpStringTsLongRenameDropped +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data* METADATA _index +| EVAL ts = TO_DATETIME(@timestamp), host_ip = TO_IP(client_ip) +| KEEP _index, event_duration, message +| SORT _index ASC, event_duration ASC +; + +_index:keyword | event_duration:long | message:keyword +sample_data | 725448 | Connection error +sample_data | 1232382 | Disconnected +sample_data | 1756467 | Connected to 10.1.0.1 +sample_data | 2764889 | Connected to 10.1.0.2 +sample_data | 3450233 | Connected to 10.1.0.3 +sample_data | 5033755 | Connection error +sample_data | 8268153 | Connection error +sample_data_str | 725448 | Connection error +sample_data_str | 1232382 | Disconnected +sample_data_str | 1756467 | Connected to 10.1.0.1 +sample_data_str | 2764889 | Connected to 10.1.0.2 +sample_data_str | 3450233 | Connected to 10.1.0.3 +sample_data_str | 5033755 | Connection error +sample_data_str | 8268153 | Connection error +sample_data_ts_long | 725448 | Connection error +sample_data_ts_long | 1232382 | Disconnected +sample_data_ts_long | 1756467 | Connected to 10.1.0.1 +sample_data_ts_long | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 5033755 | Connection error +sample_data_ts_long | 8268153 | Connection error +; + +multiIndexIpStringTsLongRenameToString +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data* METADATA _index +| EVAL ts = TO_STRING(TO_DATETIME(@timestamp)), host_ip = TO_STRING(TO_IP(client_ip)) +| KEEP _index, ts, host_ip, event_duration, message +| SORT _index ASC, ts DESC +; + +_index:keyword | ts:keyword | host_ip:keyword | event_duration:long | message:keyword +sample_data | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_str | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_str | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_str | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_str | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_str | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_str | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_str | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +sample_data_ts_long | 2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error +sample_data_ts_long | 2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error +sample_data_ts_long | 2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error +sample_data_ts_long | 2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected +sample_data_ts_long | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +sample_data_ts_long | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; + +multiIndexWhereIpStringTsLong +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data* METADATA _index +| WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) == "172.21.2.162" +| KEEP _index, event_duration, message +| SORT _index ASC, message ASC +; + +_index:keyword | event_duration:long | message:keyword +sample_data | 3450233 | Connected to 10.1.0.3 +sample_data_str | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 3450233 | Connected to 10.1.0.3 +; + +multiIndexWhereIpStringTsLongStats +required_capability: union_types + +FROM sample_data* +| WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) == "172.21.2.162" +| STATS count=count(*) BY message +| SORT count DESC, message ASC +| KEEP count, message +; + +count:long | message:keyword +3 | Connected to 10.1.0.3 +; + +multiIndexWhereIpStringLikeTsLong +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data* METADATA _index +| WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) LIKE "172.21.2.16?" +| KEEP _index, event_duration, message +| SORT _index ASC, message ASC +; + +_index:keyword | event_duration:long | message:keyword +sample_data | 3450233 | Connected to 10.1.0.3 +sample_data_str | 3450233 | Connected to 10.1.0.3 +sample_data_ts_long | 3450233 | Connected to 10.1.0.3 +; + +multiIndexWhereIpStringLikeTsLongStats +required_capability: union_types + +FROM sample_data* +| WHERE TO_LONG(@timestamp) < 1698068014937 AND TO_STRING(client_ip) LIKE "172.21.2.16?" +| STATS count=count(*) BY message +| SORT count DESC, message ASC +| KEEP count, message +; + +count:long | message:keyword +3 | Connected to 10.1.0.3 +; + +multiIndexMultiColumnTypesRename +required_capability: union_types +required_capability: metadata_fields + +FROM sample_data* METADATA _index +| WHERE event_duration > 8000000 +| EVAL ts = TO_DATETIME(@timestamp), ts_str = TO_STRING(@timestamp), ts_l = TO_LONG(@timestamp), ip = TO_IP(client_ip), ip_str = TO_STRING(client_ip) +| SORT _index ASC, ts DESC +; + +@timestamp:null | client_ip:null | event_duration:long | message:keyword | _index:keyword | ts:date | ts_str:keyword | ts_l:long | ip:ip | ip_str:k +null | null | 8268153 | Connection error | sample_data | 2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015Z | 1698069175015 | 172.21.3.15 | 172.21.3.15 +null | null | 8268153 | Connection error | sample_data_str | 2023-10-23T13:52:55.015Z | 2023-10-23T13:52:55.015Z | 1698069175015 | 172.21.3.15 | 172.21.3.15 +null | null | 8268153 | Connection error | sample_data_ts_long | 2023-10-23T13:52:55.015Z | 1698069175015 | 1698069175015 | 172.21.3.15 | 172.21.3.15 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java b/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java new file mode 100644 index 0000000000000..5b40e1d03e92f --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/test/java/org/elasticsearch/xpack/esql/CsvTestsDataLoaderTests.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql; + +import org.elasticsearch.test.ESTestCase; + +import java.net.ConnectException; + +import static org.hamcrest.Matchers.startsWith; + +public class CsvTestsDataLoaderTests extends ESTestCase { + + public void testCsvTestsDataLoaderExecution() { + ConnectException ce = expectThrows(ConnectException.class, () -> CsvTestsDataLoader.main(new String[] {})); + assertThat(ce.getMessage(), startsWith("Connection refused")); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 3eef9f7356b39..ebfcdacd7587a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -42,6 +42,11 @@ public class EsqlCapabilities { */ private static final String FN_SUBSTRING_EMPTY_NULL = "fn_substring_empty_null"; + /** + * Support for aggregation function {@code TOP_LIST}. + */ + private static final String AGG_TOP_LIST = "agg_top_list"; + /** * Optimization for ST_CENTROID changed some results in cartesian data. #108713 */ @@ -52,11 +57,6 @@ public class EsqlCapabilities { */ private static final String METADATA_IGNORED_FIELD = "metadata_field_ignored"; - /** - * Support for the "LOOKUP" command. - */ - private static final String LOOKUP_COMMAND = "lookup_command"; - /** * Support for the syntax {@code "tables": {"type": []}}. */ @@ -72,6 +72,11 @@ public class EsqlCapabilities { */ public static final String STRING_LITERAL_AUTO_CASTING_TO_DATETIME_ADD_SUB = "string_literal_auto_casting_to_datetime_add_sub"; + /** + * Support multiple field mappings if appropriate conversion function is used (union types) + */ + public static final String UNION_TYPES = "union_types"; + /** * Support for named or positional parameters in EsqlQueryRequest. */ @@ -84,14 +89,16 @@ private static Set capabilities() { caps.add(FN_CBRT); caps.add(FN_IP_PREFIX); caps.add(FN_SUBSTRING_EMPTY_NULL); + caps.add(AGG_TOP_LIST); caps.add(ST_CENTROID_AGG_OPTIMIZED); caps.add(METADATA_IGNORED_FIELD); caps.add(FN_MV_APPEND); caps.add(REPEAT); + caps.add(UNION_TYPES); caps.add(NAMED_POSITIONAL_PARAMETER); if (Build.current().isSnapshot()) { - caps.add(LOOKUP_COMMAND); + caps.add(TABLES_TYPES); } /* diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java index 2c6b5e7a6b490..4c511a4450bc8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/RequestXContent.java @@ -19,7 +19,6 @@ import org.elasticsearch.xpack.esql.parser.QueryParam; import org.elasticsearch.xpack.esql.parser.QueryParams; import org.elasticsearch.xpack.esql.plugin.QueryPragmas; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; import java.util.ArrayList; @@ -165,7 +164,7 @@ private static QueryParams parseParams(XContentParser p) throws IOException { ) ); } - type = EsqlDataTypes.fromJava(entry.getValue()); + type = DataType.fromJava(entry.getValue()); if (type == null) { errors.add(new XContentParseException(loc, entry + " is not supported as a parameter")); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 70fbe17a7d470..77a51c8415545 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -27,6 +27,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Nullability; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; @@ -59,6 +60,7 @@ import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.DateTimeArithmeticOperation; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; @@ -80,11 +82,13 @@ import org.elasticsearch.xpack.esql.stats.FeatureMetric; import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import org.elasticsearch.xpack.esql.type.MultiTypeEsField; import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -132,8 +136,13 @@ public class Analyzer extends ParameterizedRuleExecutor("Resolution", new ResolveRefs(), new ImplicitCasting()); - var finish = new Batch<>("Finish Analysis", Limiter.ONCE, new AddImplicitLimit()); + var resolution = new Batch<>( + "Resolution", + new ResolveRefs(), + new ResolveUnionTypes(), // Must be after ResolveRefs, so union types can be found + new ImplicitCasting() + ); + var finish = new Batch<>("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new UnresolveUnionTypes()); rules = List.of(init, resolution, finish); } @@ -851,14 +860,6 @@ private static List potentialCandidatesIfNoMatchesFound( } private static Attribute handleSpecialFields(UnresolvedAttribute u, Attribute named) { - if (named instanceof FieldAttribute fa) { - // incompatible mappings - var field = fa.field(); - if (field instanceof InvalidMappedField imf) { - named = u.withUnresolvedMessage("Cannot use field [" + fa.name() + "] due to ambiguities being " + imf.errorMessage()); - } - } - return named.withLocation(u.source()); } @@ -1061,4 +1062,155 @@ public static Expression castStringLiteral(Expression from, DataType target) { } } } + + /** + * The EsqlIndexResolver will create InvalidMappedField instances for fields that are ambiguous (i.e. have multiple mappings). + * During ResolveRefs we do not convert these to UnresolvedAttribute instances, as we want to first determine if they can + * instead be handled by conversion functions within the query. This rule looks for matching conversion functions and converts + * those fields into MultiTypeEsField, which encapsulates the knowledge of how to convert these into a single type. + * This knowledge will be used later in generating the FieldExtractExec with built-in type conversion. + * Any fields which could not be resolved by conversion functions will be converted to UnresolvedAttribute instances in a later rule + * (See UnresolveUnionTypes below). + */ + private static class ResolveUnionTypes extends BaseAnalyzerRule { + + record TypeResolutionKey(String fieldName, DataType fieldType) {} + + @Override + protected LogicalPlan doRule(LogicalPlan plan) { + List unionFieldAttributes = new ArrayList<>(); + // See if the eval function has an unresolved MultiTypeEsField field + // Replace the entire convert function with a new FieldAttribute (containing type conversion knowledge) + plan = plan.transformExpressionsOnly( + AbstractConvertFunction.class, + convert -> resolveConvertFunction(convert, unionFieldAttributes) + ); + // If no union fields were generated, return the plan as is + if (unionFieldAttributes.isEmpty()) { + return plan; + } + + // Otherwise drop the converted attributes after the alias function, as they are only needed for this function, and + // the original version of the attribute should still be seen as unconverted. + plan = dropConvertedAttributes(plan, unionFieldAttributes); + + // And add generated fields to EsRelation, so these new attributes will appear in the OutputExec of the Fragment + // and thereby get used in FieldExtractExec + plan = plan.transformDown(EsRelation.class, esr -> { + List output = esr.output(); + List missing = new ArrayList<>(); + for (FieldAttribute fa : unionFieldAttributes) { + if (output.stream().noneMatch(a -> a.id().equals(fa.id()))) { + missing.add(fa); + } + } + if (missing.isEmpty() == false) { + output.addAll(missing); + return new EsRelation(esr.source(), esr.index(), output, esr.indexMode(), esr.frozen()); + } + return esr; + }); + return plan; + } + + private LogicalPlan dropConvertedAttributes(LogicalPlan plan, List unionFieldAttributes) { + List projections = new ArrayList<>(plan.output()); + for (var e : unionFieldAttributes) { + projections.removeIf(p -> p.id().equals(e.id())); + } + if (projections.size() != plan.output().size()) { + return new EsqlProject(plan.source(), plan, projections); + } + return plan; + } + + private Expression resolveConvertFunction(AbstractConvertFunction convert, List unionFieldAttributes) { + if (convert.field() instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField imf) { + HashMap typeResolutions = new HashMap<>(); + Set supportedTypes = convert.supportedTypes(); + imf.getTypesToIndices().keySet().forEach(typeName -> { + DataType type = DataType.fromTypeName(typeName); + if (supportedTypes.contains(type)) { + TypeResolutionKey key = new TypeResolutionKey(fa.name(), type); + var concreteConvert = typeSpecificConvert(convert, fa.source(), type, imf); + typeResolutions.put(key, concreteConvert); + } + }); + // If all mapped types were resolved, create a new FieldAttribute with the resolved MultiTypeEsField + if (typeResolutions.size() == imf.getTypesToIndices().size()) { + var resolvedField = resolvedMultiTypeEsField(fa, typeResolutions); + return createIfDoesNotAlreadyExist(fa, resolvedField, unionFieldAttributes); + } + } else if (convert.field() instanceof AbstractConvertFunction subConvert) { + return convert.replaceChildren(Collections.singletonList(resolveConvertFunction(subConvert, unionFieldAttributes))); + } + return convert; + } + + private Expression createIfDoesNotAlreadyExist( + FieldAttribute fa, + MultiTypeEsField resolvedField, + List unionFieldAttributes + ) { + var unionFieldAttribute = new FieldAttribute(fa.source(), fa.name(), resolvedField); // Generates new ID for the field + int existingIndex = unionFieldAttributes.indexOf(unionFieldAttribute); + if (existingIndex >= 0) { + // Do not generate multiple name/type combinations with different IDs + return unionFieldAttributes.get(existingIndex); + } else { + unionFieldAttributes.add(unionFieldAttribute); + return unionFieldAttribute; + } + } + + private MultiTypeEsField resolvedMultiTypeEsField(FieldAttribute fa, HashMap typeResolutions) { + Map typesToConversionExpressions = new HashMap<>(); + InvalidMappedField imf = (InvalidMappedField) fa.field(); + imf.getTypesToIndices().forEach((typeName, indexNames) -> { + DataType type = DataType.fromTypeName(typeName); + TypeResolutionKey key = new TypeResolutionKey(fa.name(), type); + if (typeResolutions.containsKey(key)) { + typesToConversionExpressions.put(typeName, typeResolutions.get(key)); + } + }); + return MultiTypeEsField.resolveFrom(imf, typesToConversionExpressions); + } + + private Expression typeSpecificConvert(AbstractConvertFunction convert, Source source, DataType type, InvalidMappedField mtf) { + EsField field = new EsField(mtf.getName(), type, mtf.getProperties(), mtf.isAggregatable()); + NameId id = ((FieldAttribute) convert.field()).id(); + FieldAttribute resolvedAttr = new FieldAttribute(source, null, field.getName(), field, null, Nullability.TRUE, id, false); + return convert.replaceChildren(Collections.singletonList(resolvedAttr)); + } + } + + /** + * If there was no AbstractConvertFunction that resolved multi-type fields in the ResolveUnionTypes rules, + * then there could still be some FieldAttributes that contain unresolved MultiTypeEsFields. + * These need to be converted back to actual UnresolvedAttribute in order for validation to generate appropriate failures. + */ + private static class UnresolveUnionTypes extends AnalyzerRules.AnalyzerRule { + @Override + protected boolean skipResolved() { + return false; + } + + @Override + protected LogicalPlan rule(LogicalPlan plan) { + if (plan instanceof EsRelation esRelation) { + // Leave esRelation as InvalidMappedField so that UNSUPPORTED fields can still pass through + return esRelation; + } + return plan.transformExpressionsOnly(FieldAttribute.class, UnresolveUnionTypes::checkUnresolved); + } + + private static Attribute checkUnresolved(FieldAttribute fa) { + var field = fa.field(); + if (field instanceof InvalidMappedField imf) { + String unresolvedMessage = "Cannot use field [" + fa.name() + "] due to ambiguities being " + imf.errorMessage(); + return new UnresolvedAttribute(fa.source(), fa.name(), fa.qualifier(), fa.id(), unresolvedMessage, null); + } + return fa; + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java index 2b29d36cdfa1d..82eda9679074d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolver.java @@ -37,11 +37,10 @@ import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.core.index.EsIndex; -import org.elasticsearch.xpack.esql.core.index.IndexResolver; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import org.elasticsearch.xpack.esql.session.EsqlSession; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.io.IOException; @@ -359,29 +358,22 @@ public void messageReceived(LookupRequest request, TransportChannel channel, Tas } try (ThreadContext.StoredContext ignored = threadContext.stashWithOrigin(ClientHelper.ENRICH_ORIGIN)) { String indexName = EnrichPolicy.getBaseName(policyName); - indexResolver.resolveAsMergedMapping( - indexName, - IndexResolver.ALL_FIELDS, - false, - Map.of(), - refs.acquire(indexResult -> { - if (indexResult.isValid() && indexResult.get().concreteIndices().size() == 1) { - EsIndex esIndex = indexResult.get(); - var concreteIndices = Map.of(request.clusterAlias, Iterables.get(esIndex.concreteIndices(), 0)); - var resolved = new ResolvedEnrichPolicy( - p.getMatchField(), - p.getType(), - p.getEnrichFields(), - concreteIndices, - esIndex.mapping() - ); - resolvedPolices.put(policyName, resolved); - } else { - failures.put(policyName, indexResult.toString()); - } - }), - EsqlSession::specificValidity - ); + indexResolver.resolveAsMergedMapping(indexName, IndexResolver.ALL_FIELDS, refs.acquire(indexResult -> { + if (indexResult.isValid() && indexResult.get().concreteIndices().size() == 1) { + EsIndex esIndex = indexResult.get(); + var concreteIndices = Map.of(request.clusterAlias, Iterables.get(esIndex.concreteIndices(), 0)); + var resolved = new ResolvedEnrichPolicy( + p.getMatchField(), + p.getType(), + p.getEnrichFields(), + concreteIndices, + esIndex.mapping() + ); + resolvedPolices.put(policyName, resolved); + } else { + failures.put(policyName, indexResult.toString()); + } + })); } } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/QueryList.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/QueryList.java index 923b75055ca9d..417e5777d9e8c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/QueryList.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/QueryList.java @@ -15,6 +15,7 @@ import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.DoubleBlock; +import org.elasticsearch.compute.data.FloatBlock; import org.elasticsearch.compute.data.IntBlock; import org.elasticsearch.compute.data.LongBlock; import org.elasticsearch.core.Nullable; @@ -143,6 +144,10 @@ private IntFunction blockToJavaObject() { DoubleBlock doubleBlock = ((DoubleBlock) block); yield doubleBlock::getDouble; } + case FLOAT -> { + FloatBlock floatBlock = ((FloatBlock) block); + yield floatBlock::getFloat; + } case INT -> { IntBlock intBlock = (IntBlock) block; yield intBlock::getInt; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java index 7af2668e9d74b..f4979fa9928db 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java @@ -12,7 +12,6 @@ import org.elasticsearch.xpack.esql.analysis.PreAnalyzer; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.function.FunctionRegistry; -import org.elasticsearch.xpack.esql.core.index.IndexResolver; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; @@ -20,8 +19,8 @@ import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.esql.planner.Mapper; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; -import org.elasticsearch.xpack.esql.session.EsqlIndexResolver; import org.elasticsearch.xpack.esql.session.EsqlSession; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.elasticsearch.xpack.esql.stats.Metrics; import org.elasticsearch.xpack.esql.stats.QueryMetric; @@ -30,16 +29,14 @@ public class PlanExecutor { private final IndexResolver indexResolver; - private final EsqlIndexResolver esqlIndexResolver; private final PreAnalyzer preAnalyzer; private final FunctionRegistry functionRegistry; private final Mapper mapper; private final Metrics metrics; private final Verifier verifier; - public PlanExecutor(IndexResolver indexResolver, EsqlIndexResolver esqlIndexResolver) { + public PlanExecutor(IndexResolver indexResolver) { this.indexResolver = indexResolver; - this.esqlIndexResolver = esqlIndexResolver; this.preAnalyzer = new PreAnalyzer(); this.functionRegistry = new EsqlFunctionRegistry(); this.mapper = new Mapper(functionRegistry); @@ -58,7 +55,6 @@ public void esql( sessionId, cfg, indexResolver, - esqlIndexResolver, enrichPolicyResolver, preAnalyzer, functionRegistry, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java index 10800a2394e8f..11a98d3a11504 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/Order.java @@ -7,18 +7,48 @@ package org.elasticsearch.xpack.esql.expression; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.util.List; public class Order extends org.elasticsearch.xpack.esql.core.expression.Order { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Order", Order::new); + public Order(Source source, Expression child, OrderDirection direction, NullsPosition nulls) { super(source, child, direction, nulls); } + public Order(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readEnum(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.class), + in.readEnum(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.class) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + ((PlanStreamOutput) out).writeExpression(child()); + out.writeEnum(direction()); + out.writeEnum(nullsPosition()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveType() { if (DataType.isString(child().dataType())) { @@ -36,5 +66,4 @@ public Order replaceChildren(List newChildren) { protected NodeInfo info() { return NodeInfo.create(this, Order::new, child(), direction(), nullsPosition()); } - } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 8fd6ebe8d7d69..7034f23be1662 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Percentile; import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroid; import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; +import org.elasticsearch.xpack.esql.expression.function.aggregate.TopList; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -192,6 +193,7 @@ private FunctionDefinition[][] functions() { def(Min.class, Min::new, "min"), def(Percentile.class, Percentile::new, "percentile"), def(Sum.class, Sum::new, "sum"), + def(TopList.class, TopList::new, "top_list"), def(Values.class, Values::new, "values") }, // math new FunctionDefinition[] { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java index 3f6632f66bcee..1c1139c197ac0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java @@ -24,8 +24,12 @@ public class Max extends NumericAggregate implements SurrogateExpression { - @FunctionInfo(returnType = { "double", "integer", "long" }, description = "The maximum value of a numeric field.", isAggregation = true) - public Max(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { + @FunctionInfo( + returnType = { "double", "integer", "long", "date" }, + description = "The maximum value of a numeric field.", + isAggregation = true + ) + public Max(Source source, @Param(name = "number", type = { "double", "integer", "long", "date" }) Expression field) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java index 16821752bc7b8..ecfc2200a3643 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java @@ -24,8 +24,12 @@ public class Min extends NumericAggregate implements SurrogateExpression { - @FunctionInfo(returnType = { "double", "integer", "long" }, description = "The minimum value of a numeric field.", isAggregation = true) - public Min(Source source, @Param(name = "number", type = { "double", "integer", "long" }) Expression field) { + @FunctionInfo( + returnType = { "double", "integer", "long", "date" }, + description = "The minimum value of a numeric field.", + isAggregation = true + ) + public Min(Source source, @Param(name = "number", type = { "double", "integer", "long", "date" }) Expression field) { super(source, field); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java index b003b981c0709..390cd0d68018e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/NumericAggregate.java @@ -19,6 +19,28 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; +/** + * Aggregate function that receives a numeric, signed field, and returns a single double value. + *

    + * Implement the supplier methods to return the correct {@link AggregatorFunctionSupplier}. + *

    + *

    + * Some methods can be optionally overridden to support different variations: + *

    + *
      + *
    • + * {@link #supportsDates}: override to also support dates. Defaults to false. + *
      • + *
    • + * {@link #resolveType}: override to support different parameters. + * Call {@code super.resolveType()} to add extra checks. + *
      • + *
    • + * {@link #dataType}: override to return a different datatype. + * You can return {@code field().dataType()} to propagate the parameter type. + *
      • + *
    + */ public abstract class NumericAggregate extends AggregateFunction implements ToAggregator { NumericAggregate(Source source, Expression field, List parameters) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopList.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopList.java new file mode 100644 index 0000000000000..79893b1c7de07 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/TopList.java @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.compute.aggregation.AggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.TopListDoubleAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.TopListIntAggregatorFunctionSupplier; +import org.elasticsearch.compute.aggregation.TopListLongAggregatorFunctionSupplier; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.SurrogateExpression; +import org.elasticsearch.xpack.esql.expression.function.Example; +import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; +import org.elasticsearch.xpack.esql.expression.function.Param; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.planner.ToAggregator; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isFoldable; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; + +public class TopList extends AggregateFunction implements ToAggregator, SurrogateExpression { + private static final String ORDER_ASC = "ASC"; + private static final String ORDER_DESC = "DESC"; + + @FunctionInfo( + returnType = { "double", "integer", "long", "date" }, + description = "Collects the top values for a field. Includes repeated values.", + isAggregation = true, + examples = @Example(file = "stats_top_list", tag = "top-list") + ) + public TopList( + Source source, + @Param( + name = "field", + type = { "double", "integer", "long", "date" }, + description = "The field to collect the top values for." + ) Expression field, + @Param(name = "limit", type = { "integer" }, description = "The maximum number of values to collect.") Expression limit, + @Param( + name = "order", + type = { "keyword" }, + description = "The order to calculate the top values. Either `asc` or `desc`." + ) Expression order + ) { + super(source, field, Arrays.asList(limit, order)); + } + + public static TopList readFrom(PlanStreamInput in) throws IOException { + return new TopList(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readExpression()); + } + + public void writeTo(PlanStreamOutput out) throws IOException { + source().writeTo(out); + List fields = children(); + assert fields.size() == 3; + out.writeExpression(fields.get(0)); + out.writeExpression(fields.get(1)); + out.writeExpression(fields.get(2)); + } + + private Expression limitField() { + return parameters().get(0); + } + + private Expression orderField() { + return parameters().get(1); + } + + private int limitValue() { + return (int) limitField().fold(); + } + + private String orderRawValue() { + return BytesRefs.toString(orderField().fold()); + } + + private boolean orderValue() { + return orderRawValue().equalsIgnoreCase(ORDER_ASC); + } + + @Override + protected TypeResolution resolveType() { + if (childrenResolved() == false) { + return new TypeResolution("Unresolved children"); + } + + var typeResolution = isType( + field(), + dt -> dt == DataType.DATETIME || dt.isNumeric() && dt != DataType.UNSIGNED_LONG, + sourceText(), + FIRST, + "numeric except unsigned_long or counter types" + ).and(isFoldable(limitField(), sourceText(), SECOND)) + .and(isType(limitField(), dt -> dt == DataType.INTEGER, sourceText(), SECOND, "integer")) + .and(isFoldable(orderField(), sourceText(), THIRD)) + .and(isString(orderField(), sourceText(), THIRD)); + + if (typeResolution.unresolved()) { + return typeResolution; + } + + var limit = limitValue(); + var order = orderRawValue(); + + if (limit <= 0) { + return new TypeResolution(format(null, "Limit must be greater than 0 in [{}], found [{}]", sourceText(), limit)); + } + + if (order.equalsIgnoreCase(ORDER_ASC) == false && order.equalsIgnoreCase(ORDER_DESC) == false) { + return new TypeResolution( + format(null, "Invalid order value in [{}], expected [{}, {}] but got [{}]", sourceText(), ORDER_ASC, ORDER_DESC, order) + ); + } + + return TypeResolution.TYPE_RESOLVED; + } + + @Override + public DataType dataType() { + return field().dataType(); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, TopList::new, children().get(0), children().get(1), children().get(2)); + } + + @Override + public TopList replaceChildren(List newChildren) { + return new TopList(source(), newChildren.get(0), newChildren.get(1), newChildren.get(2)); + } + + @Override + public AggregatorFunctionSupplier supplier(List inputChannels) { + DataType type = field().dataType(); + if (type == DataType.LONG || type == DataType.DATETIME) { + return new TopListLongAggregatorFunctionSupplier(inputChannels, limitValue(), orderValue()); + } + if (type == DataType.INTEGER) { + return new TopListIntAggregatorFunctionSupplier(inputChannels, limitValue(), orderValue()); + } + if (type == DataType.DOUBLE) { + return new TopListDoubleAggregatorFunctionSupplier(inputChannels, limitValue(), orderValue()); + } + throw EsqlIllegalArgumentException.illegalDataType(type); + } + + @Override + public Expression surrogate() { + var s = source(); + + if (limitValue() == 1) { + if (orderValue()) { + return new Min(s, field()); + } else { + return new Max(s, field()); + } + } + + return null; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java new file mode 100644 index 0000000000000..a99c7a8b7ac8d --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/package-info.java @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +/** + * Functions that aggregate values, with or without grouping within buckets. + * Used in `STATS` and similar commands. + * + *

    Guide to adding new aggregate function

    + *
      + *
    1. + * Aggregation functions are more complex than scalar functions, so it's a good idea to discuss + * the new function with the ESQL team before starting to implement it. + *

      + * You may also discuss its implementation, as aggregations may require special performance considerations. + *

      + *
      2. + *
    2. + * To learn the basics about making functions, check {@link org.elasticsearch.xpack.esql.expression.function.scalar}. + *

      + * It has the guide to making a simple function, which should be a good base to start doing aggregations. + *

      + *
      3. + *
    3. + * Pick one of the csv-spec files in {@code x-pack/plugin/esql/qa/testFixtures/src/main/resources/} + * and add a test for the function you want to write. These files are roughly themed but there + * isn't a strong guiding principle in the organization. + *
      4. + *
    4. + * Rerun the {@code CsvTests} and watch your new test fail. + *
      5. + *
    5. + * Find an aggregate function in this package similar to the one you are working on and copy it to build + * yours. + * Your function might extend from the available abstract classes. Check the javadoc of each before using them: + *
        + *
      • + * {@link org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction}: The base class for aggregates + *
        • + *
      • + * {@link org.elasticsearch.xpack.esql.expression.function.aggregate.NumericAggregate}: Aggregation for numeric values + *
        • + *
      • + * {@link org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialAggregateFunction}: + * Aggregation for spatial values + *
        • + *
      + *
      6. + *
    6. + * Fill the required methods in your new function. Check their JavaDoc for more information. + * Here are some of the important ones: + *
        + *
      • + * Constructor: Review the constructor annotations, and make sure to add the correct types and descriptions. + *
          + *
        • {@link org.elasticsearch.xpack.esql.expression.function.FunctionInfo}, for the constructor itself
          • + *
        • {@link org.elasticsearch.xpack.esql.expression.function.Param}, for the function parameters
          • + *
        + *
        • + *
      • + * {@code resolveType}: Check the metadata of your function parameters. + * This may include types, whether they are foldable or not, or their possible values. + *
        • + *
      • + * {@code dataType}: This will return the datatype of your function. + * May be based on its current parameters. + *
        • + *
      + * + * Finally, you may want to implement some interfaces. + * Check their JavaDocs to see if they are suitable for your function: + *
        + *
      • + * {@link org.elasticsearch.xpack.esql.planner.ToAggregator}: (More information about aggregators below) + *
        • + *
      • + * {@link org.elasticsearch.xpack.esql.expression.SurrogateExpression} + *
        • + *
      + *
      7. + *
    7. + * To introduce your aggregation to the engine: + *
        + *
      • + * Add it to {@code org.elasticsearch.xpack.esql.planner.AggregateMapper}. + * Check all usages of other aggregations there, and replicate the logic. + *
        • + *
      • + * Add it to {@link org.elasticsearch.xpack.esql.io.stream.PlanNamedTypes}. + * Consider adding a {@code writeTo} method and a constructor/{@code readFrom} method inside your function, + * to keep all the logic in one place. + *

        + * You can find examples of other aggregations using this method, + * like {@link org.elasticsearch.xpack.esql.expression.function.aggregate.TopList#writeTo(PlanStreamOutput)} + *

        + *
        • + *
      • + * Do the same with {@link org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry}. + *
        • + *
      + *
      8. + *
    + * + *

    Creating aggregators for your function

    + *

    + * Aggregators contain the core logic of your aggregation. That is, how to combine values, what to store, how to process data, etc. + *

    + *
      + *
    1. + * Copy an existing aggregator to use as a base. You'll usually make one per type. Check other classes to see the naming pattern. + * You can find them in {@link org.elasticsearch.compute.aggregation}. + *

      + * Note that some aggregators are autogenerated, so they live in different directories. + * The base is {@code x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/} + *

      + *
      2. + *
    2. + * Make a test for your aggregator. + * You can copy an existing one from {@code x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/aggregation/}. + *

      + * Tests extending from {@code org.elasticsearch.compute.aggregation.AggregatorFunctionTestCase} + * will already include most required cases. You should only need to fill the required abstract methods. + *

      + *
      3. + *
    3. + * Check the Javadoc of the {@link org.elasticsearch.compute.ann.Aggregator} + * and {@link org.elasticsearch.compute.ann.GroupingAggregator} annotations. + * Add/Modify them on your aggregator. + *
      4. + *
    4. + * The {@link org.elasticsearch.compute.ann.Aggregator} JavaDoc explains the static methods you should add. + *
      5. + *
    5. + * After implementing the required methods (Even if they have a dummy implementation), + * run the CsvTests to generate some extra required classes. + *

      + * One of them will be the {@code AggregatorFunctionSupplier} for your aggregator. + * Find it by its name ({@code AggregatorFunctionSupplier}), + * and return it in the {@code toSupplier} method in your function, under the correct type condition. + *

      + *
      6. + *
    6. + * Now, complete the implementation of the aggregator, until the tests pass! + *
      7. + *
    + * + *

    StringTemplates

    + *

    + * Making an aggregator per type may be repetitive. To avoid code duplication, we use StringTemplates: + *

    + *
      + *
    1. + * Create a new StringTemplate file. + * Use another as a reference, like + * {@code x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/aggregation/X-TopListAggregator.java.st}. + *
      2. + *
    2. + * Add the template scripts to {@code x-pack/plugin/esql/compute/build.gradle}. + *

      + * You can also see there which variables you can use, and which types are currently supported. + *

      + *
      3. + *
    3. + * After completing your template, run the generation with {@code ./gradlew :x-pack:plugin:esql:compute:compileJava}. + *

      + * You may need to tweak some import orders per type so they don't raise warnings. + *

      + *
      4. + *
    + */ +package org.elasticsearch.xpack.esql.expression.function.aggregate; + +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java index 4f991af54ecff..17934c1729ad7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/EsqlScalarFunction.java @@ -7,10 +7,25 @@ package org.elasticsearch.xpack.esql.expression.function.scalar; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateDiff; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateFormat; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateParse; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; +import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; +import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals; import java.util.List; @@ -25,6 +40,24 @@ *

    */ public abstract class EsqlScalarFunction extends ScalarFunction implements EvaluatorMapper { + public static List getNamedWriteables() { + return List.of( + Case.ENTRY, + Coalesce.ENTRY, + Concat.ENTRY, + Greatest.ENTRY, + InsensitiveEquals.ENTRY, + DateExtract.ENTRY, + DateDiff.ENTRY, + DateFormat.ENTRY, + DateParse.ENTRY, + DateTrunc.ENTRY, + Least.ENTRY, + Now.ENTRY, + ToLower.ENTRY, + ToUpper.ENTRY + ); + } protected EsqlScalarFunction(Source source) { super(source); @@ -38,5 +71,4 @@ protected EsqlScalarFunction(Source source, List fields) { public Object fold() { return EvaluatorMapper.super.fold(); } - } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java index 0866f97b67724..eb2e5ab94487f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/UnaryScalarFunction.java @@ -12,6 +12,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FromBase64; @@ -76,10 +79,13 @@ public static List getNamedWriteables() { Cosh.ENTRY, Floor.ENTRY, FromBase64.ENTRY, + IsNotNull.ENTRY, + IsNull.ENTRY, Length.ENTRY, Log10.ENTRY, LTrim.ENTRY, Neg.ENTRY, + Not.ENTRY, RTrim.ENTRY, Signum.ENTRY, Sin.ENTRY, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index f98f5c45acd16..50d0e5484756e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.ElementType; @@ -27,8 +30,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -37,8 +43,12 @@ import static org.elasticsearch.common.logging.LoggerMessageFormat.format; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; public final class Case extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Case", Case::new); + record Condition(Expression condition, Expression value) {} private final List conditions; @@ -110,6 +120,26 @@ public Case( elseValue = elseValueIsExplicit() ? children().get(children().size() - 1) : new Literal(source, null, NULL); } + private Case(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + private boolean elseValueIsExplicit() { return children().size() % 2 == 1; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java index 8062019b4c51c..580e2f9900208 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Greatest.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -23,17 +26,24 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMax; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.util.List; import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * Returns the maximum value of multiple columns. */ public class Greatest extends EsqlScalarFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Greatest", Greatest::new); + private DataType dataType; @FunctionInfo( @@ -61,6 +71,26 @@ public Greatest( super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } + private Greatest(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { if (dataType == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java index f983e0125a4db..2255fed9d4947 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Least.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -23,17 +26,24 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.util.List; import java.util.function.Function; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * Returns the minimum value of multiple columns. */ public class Least extends EsqlScalarFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Least", Least::new); + private DataType dataType; @FunctionInfo( @@ -59,6 +69,26 @@ public Least( super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } + private Least(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { if (dataType == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java index 2496d8b82fa6f..96601905d40c9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/AbstractConvertFunction.java @@ -77,7 +77,11 @@ protected final TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } - return isType(field(), factories()::containsKey, sourceText(), null, supportedTypesNames(factories().keySet())); + return isType(field(), factories()::containsKey, sourceText(), null, supportedTypesNames(supportedTypes())); + } + + public Set supportedTypes() { + return factories().keySet(); } public static String supportedTypesNames(Set types) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java index 42e20a9a4615e..2a224598253f9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiff.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -20,7 +23,10 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -47,6 +53,7 @@ * If the second argument (start) is greater than the third argument (end), then negative values are returned. */ public class DateDiff extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "DateDiff", DateDiff::new); public static final ZoneId UTC = ZoneId.of("Z"); @@ -166,6 +173,40 @@ public DateDiff( this.endTimestamp = endTimestamp; } + private DateDiff(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + ((PlanStreamOutput) out).writeExpression(unit); + ((PlanStreamOutput) out).writeExpression(startTimestamp); + ((PlanStreamOutput) out).writeExpression(endTimestamp); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + Expression unit() { + return unit; + } + + Expression startTimestamp() { + return startTimestamp; + } + + Expression endTimestamp() { + return endTimestamp; + } + @Evaluator(extraName = "Constant", warnExceptions = { IllegalArgumentException.class, InvalidArgumentException.class }) static int process(@Fixed Part datePartFieldUnit, long startTimestamp, long endTimestamp) throws IllegalArgumentException { ZonedDateTime zdtStart = ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTimestamp), UTC); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java index c28c5e417c152..f3448a2b7c5ff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtract.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -22,8 +25,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.time.ZoneId; import java.time.temporal.ChronoField; import java.util.List; @@ -35,6 +41,11 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.chronoToLong; public class DateExtract extends EsqlConfigurationFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "DateExtract", + DateExtract::new + ); private ChronoField chronoField; @@ -69,6 +80,35 @@ public DateExtract( super(source, List.of(chronoFieldExp, field), configuration); } + private DateExtract(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).configuration() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(datePart()); + ((PlanStreamOutput) out).writeExpression(field()); + } + + Expression datePart() { + return children().get(0); + } + + Expression field() { + return children().get(1); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public ExpressionEvaluator.Factory toEvaluator(Function toEvaluator) { var fieldEvaluator = toEvaluator.apply(children().get(1)); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java index bcc5d7cb16050..9a789c2bb6fb2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormat.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -22,9 +25,12 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -37,6 +43,11 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; public class DateFormat extends EsqlConfigurationFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "DateFormat", + DateFormat::new + ); private final Expression field; private final Expression format; @@ -59,6 +70,35 @@ Date format (optional). If no format is specified, the `yyyy-MM-dd'T'HH:mm:ss.S this.format = date != null ? format : null; } + private DateFormat(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readOptionalWriteable(i -> ((PlanStreamInput) i).readExpression()), + ((PlanStreamInput) in).configuration() + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeOptionalWriteable(children().size() == 1 ? null : o -> ((PlanStreamOutput) o).writeExpression(children().get(1))); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + Expression field() { + return field; + } + + Expression format() { + return format; + } + @Override public DataType dataType() { return DataType.KEYWORD; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java index d68664afe8418..12ffe092287ed 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParse.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -22,8 +25,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.time.ZoneId; import java.util.List; import java.util.function.Function; @@ -38,6 +44,11 @@ import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToLong; public class DateParse extends EsqlScalarFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "DateParse", + DateParse::new + ); private final Expression field; private final Expression format; @@ -64,6 +75,26 @@ public DateParse( this.format = second != null ? first : null; } + private DateParse(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readOptionalWriteable(i -> ((PlanStreamInput) i).readExpression()) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeOptionalWriteable(children().size() == 2 ? o -> ((PlanStreamOutput) out).writeExpression(children().get(1)) : null); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { return DataType.DATETIME; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java index ddd51d281105d..995e525dda9ec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; import org.elasticsearch.common.Rounding; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -20,8 +23,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.time.Duration; import java.time.Period; import java.time.ZoneId; @@ -36,6 +42,12 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; public class DateTrunc extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "DateTrunc", + DateTrunc::new + ); + private final Expression interval; private final Expression timestampField; protected static final ZoneId DEFAULT_TZ = ZoneOffset.UTC; @@ -69,6 +81,30 @@ public DateTrunc( this.timestampField = field; } + private DateTrunc(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), ((PlanStreamInput) in).readExpression(), ((PlanStreamInput) in).readExpression()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(interval); + ((PlanStreamOutput) out).writeExpression(timestampField); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + Expression interval() { + return interval; + } + + Expression field() { + return timestampField; + } + @Override protected TypeResolution resolveType() { if (childrenResolved() == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java index fe54cfd186fec..0f401e3de8045 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/Now.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.date; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -18,11 +21,14 @@ import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import java.io.IOException; import java.util.List; import java.util.function.Function; public class Now extends EsqlConfigurationFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Now", Now::new); private final long now; @@ -38,13 +44,18 @@ public Now(Source source, Configuration configuration) { this.now = configuration.now() == null ? System.currentTimeMillis() : configuration.now().toInstant().toEpochMilli(); } - private Now(Source source, long now) { - super(source, List.of(), null); - this.now = now; + private Now(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), ((PlanStreamInput) in).configuration()); } - public static Now newInstance(Source source, long now) { - return new Now(source, now); + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + } + + @Override + public String getWriteableName() { + return ENTRY.name; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java index 5aa6dad7b2a5b..9b7e0b729cde9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunction.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.operator.DriverContext; @@ -15,7 +18,12 @@ import org.elasticsearch.core.Releasables; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.List; /** * Base class for functions that reduce multivalued fields into single valued fields. @@ -25,10 +33,39 @@ *

    */ public abstract class AbstractMultivalueFunction extends UnaryScalarFunction { + public static List getNamedWriteables() { + return List.of( + MvAppend.ENTRY, + MvAvg.ENTRY, + MvConcat.ENTRY, + MvCount.ENTRY, + MvDedupe.ENTRY, + MvFirst.ENTRY, + MvLast.ENTRY, + MvMax.ENTRY, + MvMedian.ENTRY, + MvMin.ENTRY, + MvSlice.ENTRY, + MvSort.ENTRY, + MvSum.ENTRY, + MvZip.ENTRY + ); + } + protected AbstractMultivalueFunction(Source source, Expression field) { super(source, field); } + protected AbstractMultivalueFunction(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), ((PlanStreamInput) in).readExpression()); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + ((PlanStreamOutput) out).writeExpression(field); + } + /** * Build the evaluator given the evaluator a multivalued field. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java index 1f37c15ecfc43..99844d40e0565 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; @@ -25,9 +28,11 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Objects; @@ -41,6 +46,8 @@ * Appends values to a multi-value */ public class MvAppend extends EsqlScalarFunction implements EvaluatorMapper { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvAppend", MvAppend::new); + private final Expression field1, field2; private DataType dataType; @@ -103,6 +110,22 @@ public MvAppend( this.field2 = field2; } + private MvAppend(StreamInput in) throws IOException { + this(Source.readFrom((PlanStreamInput) in), ((PlanStreamInput) in).readExpression(), ((PlanStreamInput) in).readExpression()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + out.writeNamedWriteable(field1); + out.writeNamedWriteable(field2); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveType() { if (childrenResolved() == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java index 787bf3e5efd1c..01f24365be225 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvg.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.MvEvaluator; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -21,6 +23,7 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -31,6 +34,8 @@ * Reduce a multivalued field to a single valued field containing the average value. */ public class MvAvg extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvAvg", MvAvg::new); + @FunctionInfo( returnType = "double", description = "Converts a multivalued field into a single valued field containing the average of all of the values.", @@ -47,6 +52,15 @@ public MvAvg( super(source, field); } + private MvAvg(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), t -> t.isNumeric() && isRepresentable(t), sourceText(), null, "numeric"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java index 3e37a739147cf..fa9475055515f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcat.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.data.Page; @@ -25,6 +27,7 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; +import java.io.IOException; import java.util.function.Function; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; @@ -33,6 +36,8 @@ * Reduce a multivalued string field to a single valued field by concatenating all values. */ public class MvConcat extends BinaryScalarFunction implements EvaluatorMapper { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvConcat", MvConcat::new); + @FunctionInfo( returnType = "keyword", description = "Converts a multivalued string expression into a single valued column " @@ -53,6 +58,15 @@ public MvConcat( super(source, field, delim); } + private MvConcat(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveType() { if (childrenResolved() == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java index b2afef4f2235e..faf7d36e4a24c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.operator.DriverContext; import org.elasticsearch.compute.operator.EvalOperator; @@ -20,6 +22,7 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -28,6 +31,8 @@ * Reduce a multivalued field to a single valued field containing the count of values. */ public class MvCount extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvCount", MvCount::new); + @FunctionInfo( returnType = "integer", description = "Converts a multivalued expression into a single valued column containing a count of the number of values.", @@ -58,6 +63,15 @@ public MvCount( super(source, v); } + private MvCount(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java index 71cf759b3dbe5..d17bc26ab808b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.compute.operator.mvdedupe.MultivalueDedupe; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -18,6 +20,7 @@ import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -26,6 +29,8 @@ * Removes duplicate values from a multivalued field. */ public class MvDedupe extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvDedupe", MvDedupe::new); + // @TODO: add unsigned_long @FunctionInfo( returnType = { @@ -70,6 +75,15 @@ public MvDedupe( super(source, field); } + private MvDedupe(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java index a985c10824ae7..25e6a85a485c1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.MvEvaluator; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; @@ -26,6 +28,7 @@ import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -34,6 +37,8 @@ * Reduce a multivalued field to a single valued field containing the minimum value. */ public class MvFirst extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvFirst", MvFirst::new); + @FunctionInfo( returnType = { "boolean", @@ -87,6 +92,15 @@ public MvFirst( super(source, field); } + private MvFirst(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java index 8dcc4c8b1222e..2a9a498ecf9d3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.MvEvaluator; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; @@ -26,6 +28,7 @@ import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -34,6 +37,8 @@ * Reduce a multivalued field to a single valued field containing the minimum value. */ public class MvLast extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvLast", MvLast::new); + @FunctionInfo( returnType = { "boolean", @@ -87,6 +92,15 @@ public MvLast( super(source, field); } + private MvLast(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), EsqlDataTypes::isRepresentable, sourceText(), null, "representable"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java index 7cfc4a94b35d4..24873cc1da2e9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.MvEvaluator; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -20,6 +22,7 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -30,6 +33,8 @@ * Reduce a multivalued field to a single valued field containing the maximum value. */ public class MvMax extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvMax", MvMax::new); + @FunctionInfo( returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, description = "Converts a multivalued expression into a single valued column containing the maximum value.", @@ -53,6 +58,15 @@ public MvMax( super(source, v); } + private MvMax(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), t -> isSpatial(t) == false && isRepresentable(t), sourceText(), null, "representableNonSpatial"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java index 8d3177926f2e6..4e7d6dd4e29b2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedian.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.MvEvaluator; import org.elasticsearch.compute.data.DoubleBlock; import org.elasticsearch.compute.data.IntBlock; @@ -23,6 +25,7 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.math.BigInteger; import java.util.Arrays; import java.util.List; @@ -36,6 +39,8 @@ * Reduce a multivalued field to a single valued field containing the average value. */ public class MvMedian extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvMedian", MvMedian::new); + @FunctionInfo( returnType = { "double", "integer", "long", "unsigned_long" }, description = "Converts a multivalued field into a single valued field containing the median value.", @@ -60,6 +65,15 @@ public MvMedian( super(source, field); } + private MvMedian(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), t -> t.isNumeric() && isRepresentable(t), sourceText(), null, "numeric"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java index e52e72c766a3d..205a09953fde3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java @@ -8,6 +8,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.MvEvaluator; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -20,6 +22,7 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -30,6 +33,8 @@ * Reduce a multivalued field to a single valued field containing the minimum value. */ public class MvMin extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvMin", MvMin::new); + @FunctionInfo( returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" }, description = "Converts a multivalued expression into a single valued column containing the minimum value.", @@ -53,6 +58,15 @@ public MvMin( super(source, field); } + private MvMin(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), t -> isSpatial(t) == false && isRepresentable(t), sourceText(), null, "representableNonSpatial"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index 40e9f90df9dc6..f824d0821cfbf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; @@ -23,14 +26,17 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.function.Function; @@ -46,6 +52,8 @@ * Returns a subset of the multivalued field using the start and end index values. */ public class MvSlice extends EsqlScalarFunction implements OptionalArgument, EvaluatorMapper { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvSlice", MvSlice::new); + private final Expression field, start, end; @FunctionInfo( @@ -103,7 +111,43 @@ public MvSlice( super(source, end == null ? Arrays.asList(field, start, start) : Arrays.asList(field, start, end)); this.field = field; this.start = start; - this.end = end == null ? start : end; + this.end = end; + } + + private MvSlice(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression(), + // TODO readOptionalNamedWriteable + in.readOptionalWriteable(i -> ((PlanStreamInput) i).readExpression()) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + ((PlanStreamOutput) out).writeExpression(field); + ((PlanStreamOutput) out).writeExpression(start); + // TODO writeOptionalNamedWriteable + out.writeOptionalWriteable(end == null ? null : o -> ((PlanStreamOutput) o).writeExpression(end)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + Expression field() { + return field; + } + + Expression start() { + return start; + } + + Expression end() { + return end; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index 744491b30f702..fd5f493ae405e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -9,6 +9,9 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; import org.elasticsearch.compute.data.BooleanBlock; @@ -33,13 +36,16 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.function.Function; @@ -54,6 +60,8 @@ * Sorts a multivalued field in lexicographical order. */ public class MvSort extends EsqlScalarFunction implements OptionalArgument, Validatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvSort", MvSort::new); + private final Expression field, order; private static final Literal ASC = new Literal(Source.EMPTY, "ASC", DataType.KEYWORD); @@ -79,7 +87,37 @@ public MvSort( ) { super(source, order == null ? Arrays.asList(field, ASC) : Arrays.asList(field, order)); this.field = field; - this.order = order == null ? ASC : order; + this.order = order; + } + + private MvSort(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + // TODO readOptionalNamedWriteable + in.readOptionalWriteable(i -> ((PlanStreamInput) i).readExpression()) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(field); + // TODO writeOptionalNamedWriteable + out.writeOptionalWriteable(order == null ? null : o -> ((PlanStreamOutput) o).writeExpression(order)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + Expression field() { + return field; + } + + Expression order() { + return order; } @Override @@ -93,6 +131,9 @@ protected TypeResolution resolveType() { if (resolution.unresolved()) { return resolution; } + if (order == null) { + return resolution; + } return isString(order, sourceText(), SECOND); } @@ -106,7 +147,10 @@ public boolean foldable() { public EvalOperator.ExpressionEvaluator.Factory toEvaluator( Function toEvaluator ) { - boolean ordering = order.foldable() && ((BytesRef) order.fold()).utf8ToString().equalsIgnoreCase("DESC") ? false : true; + Expression nonNullOrder = order == null ? ASC : order; + boolean ordering = nonNullOrder.foldable() && ((BytesRef) nonNullOrder.fold()).utf8ToString().equalsIgnoreCase("DESC") + ? false + : true; return switch (PlannerUtils.toElementType(field.dataType())) { case BOOLEAN -> new MvSort.EvaluatorFactory( toEvaluator.apply(field), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java index e14bc401a058a..eabf5e20ad1b0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSum.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.MvEvaluator; import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; @@ -21,6 +23,7 @@ import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; @@ -31,6 +34,8 @@ * Reduce a multivalued field to a single valued field containing the sum of all values. */ public class MvSum extends AbstractMultivalueFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvSum", MvSum::new); + @FunctionInfo( returnType = { "double", "integer", "long", "unsigned_long" }, description = "Converts a multivalued field into a single valued field containing the sum of all of the values.", @@ -47,6 +52,15 @@ public MvSum( super(source, field); } + private MvSum(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveFieldType() { return isType(field(), t -> t.isNumeric() && isRepresentable(t), sourceText(), null, "numeric"); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java index 4f42858cbedba..15bd09a4089e6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZip.java @@ -9,6 +9,9 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.data.BytesRefBlock; import org.elasticsearch.compute.operator.EvalOperator; @@ -19,12 +22,15 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.function.Function; @@ -38,6 +44,8 @@ * Combines the values from two multivalued fields with a delimiter that joins them together. */ public class MvZip extends EsqlScalarFunction implements OptionalArgument, EvaluatorMapper { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "MvZip", MvZip::new); + private final Expression mvLeft, mvRight, delim; private static final Literal COMMA = new Literal(Source.EMPTY, ",", DataType.TEXT); @@ -60,7 +68,31 @@ public MvZip( super(source, delim == null ? Arrays.asList(mvLeft, mvRight, COMMA) : Arrays.asList(mvLeft, mvRight, delim)); this.mvLeft = mvLeft; this.mvRight = mvRight; - this.delim = delim == null ? COMMA : delim; + this.delim = delim; + } + + private MvZip(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression(), + // TODO readOptionalNamedWriteable + in.readOptionalWriteable(i -> ((PlanStreamInput) i).readExpression()) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + Source.EMPTY.writeTo(out); + ((PlanStreamOutput) out).writeExpression(mvLeft); + ((PlanStreamOutput) out).writeExpression(mvRight); + // TODO writeOptionalNamedWriteable + out.writeOptionalWriteable(delim == null ? null : o -> ((PlanStreamOutput) o).writeExpression(delim)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; } @Override @@ -104,7 +136,12 @@ public Nullability nullable() { public EvalOperator.ExpressionEvaluator.Factory toEvaluator( Function toEvaluator ) { - return new MvZipEvaluator.Factory(source(), toEvaluator.apply(mvLeft), toEvaluator.apply(mvRight), toEvaluator.apply(delim)); + return new MvZipEvaluator.Factory( + source(), + toEvaluator.apply(mvLeft), + toEvaluator.apply(mvRight), + toEvaluator.apply(delim == null ? COMMA : delim) + ); } @Override @@ -195,4 +232,16 @@ static void process(BytesRefBlock.Builder builder, int position, BytesRefBlock l } builder.endPositionEntry(); } + + Expression mvLeft() { + return mvLeft; + } + + Expression mvRight() { + return mvRight; + } + + Expression delim() { + return delim; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java index ff7cd83eedbe2..6a02eb4b94f12 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/Coalesce.java @@ -7,6 +7,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.ElementType; import org.elasticsearch.compute.data.Page; @@ -27,19 +30,26 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.planner.PlannerUtils; +import java.io.IOException; import java.util.List; import java.util.function.Function; import java.util.stream.IntStream; import java.util.stream.Stream; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * Function returning the first non-null value. */ public class Coalesce extends EsqlScalarFunction implements OptionalArgument { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Coalesce", Coalesce::new); + private DataType dataType; @FunctionInfo( @@ -100,6 +110,26 @@ public Coalesce( super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } + private Coalesce(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { if (dataType == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java index 7e7a024ba2c4e..2e40ee1634d1b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/package-info.java @@ -110,7 +110,9 @@ * Register your function for serialization. We're in the process of migrating this serialization * from an older way to the more common, {@link org.elasticsearch.common.io.stream.NamedWriteable}. *

    - * All subclasses of {@link org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction} + * All subclasses of {@link org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction}, + * {@link org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison}, + * and {@link org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation} * are migrated and should include a "getWriteableName", "writeTo", and a deserializing constructor. * They should also include a {@link org.elasticsearch.common.io.stream.NamedWriteableRegistry.Entry} * and it should be linked in {@link org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction}. @@ -131,7 +133,7 @@ * *

  • * Now it's time to make a unit test! The infrastructure for these is under some flux at - * the moment, but it's good to extend from {@code AbstractScalarFunctionTestCase}. All of + * the moment, but it's good to extend from {@code AbstractFunctionTestCase}. All of * these tests are parameterized and expect to spend some time finding good parameters. *
    • *
  • diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java index d01edbe7024e8..69464787f9288 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/Concat.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.compute.operator.BreakingBytesRefBuilder; @@ -22,7 +25,10 @@ import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import java.io.IOException; import java.util.List; import java.util.function.Function; import java.util.stream.Stream; @@ -30,11 +36,14 @@ import static org.elasticsearch.common.unit.ByteSizeUnit.MB; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; +import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; /** * Join strings. */ public class Concat extends EsqlScalarFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Concat", Concat::new); static final long MAX_CONCAT_LENGTH = MB.toBytes(1); @@ -51,6 +60,26 @@ public Concat( super(source, Stream.concat(Stream.of(first), rest.stream()).toList()); } + private Concat(StreamInput in) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) + ); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + ((PlanStreamOutput) out).writeExpression(children().get(0)); + out.writeCollection(children().subList(1, children().size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { return DataType.KEYWORD; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java index f14df4f56929a..aadb0b3ac7886 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLower.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -17,12 +20,15 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; +import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -31,6 +37,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; public class ToLower extends EsqlConfigurationFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "ToLower", ToLower::new); private final Expression field; @@ -52,6 +59,20 @@ public ToLower( this.field = field; } + private ToLower(StreamInput in) throws IOException { + this(Source.EMPTY, ((PlanStreamInput) in).readExpression(), ((PlanStreamInput) in).configuration()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ((PlanStreamOutput) out).writeExpression(field()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { return field.dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java index 6c903b4bfddeb..398fe1c76a49f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpper.java @@ -8,6 +8,9 @@ package org.elasticsearch.xpack.esql.expression.function.scalar.string; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; @@ -17,12 +20,15 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.util.PlanStreamOutput; import org.elasticsearch.xpack.esql.expression.function.Example; import org.elasticsearch.xpack.esql.expression.function.FunctionInfo; import org.elasticsearch.xpack.esql.expression.function.Param; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlConfigurationFunction; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; +import java.io.IOException; import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -31,6 +37,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; public class ToUpper extends EsqlConfigurationFunction { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "ToUpper", ToUpper::new); private final Expression field; @@ -52,6 +59,20 @@ public ToUpper( this.field = field; } + private ToUpper(StreamInput in) throws IOException { + this(Source.EMPTY, ((PlanStreamInput) in).readExpression(), ((PlanStreamInput) in).configuration()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + ((PlanStreamOutput) out).writeExpression(field()); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { return field.dataType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java index b84082d410af3..fcf71900c4198 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Add.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -14,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import java.io.IOException; import java.time.DateTimeException; import java.time.Duration; import java.time.Period; @@ -25,6 +28,7 @@ import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation.OperationSymbol.ADD; public class Add extends DateTimeArithmeticOperation implements BinaryComparisonInversible { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Add", Add::new); public Add(Source source, Expression left, Expression right) { super( @@ -35,11 +39,28 @@ public Add(Source source, Expression left, Expression right) { AddIntsEvaluator.Factory::new, AddLongsEvaluator.Factory::new, AddUnsignedLongsEvaluator.Factory::new, - (s, lhs, rhs) -> new AddDoublesEvaluator.Factory(source, lhs, rhs), + AddDoublesEvaluator.Factory::new, AddDatetimesEvaluator.Factory::new ); } + private Add(StreamInput in) throws IOException { + super( + in, + ADD, + AddIntsEvaluator.Factory::new, + AddLongsEvaluator.Factory::new, + AddUnsignedLongsEvaluator.Factory::new, + AddDoublesEvaluator.Factory::new, + AddDatetimesEvaluator.Factory::new + ); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, Add::new, left(), right()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index 04a7b8a6067bd..45cc5b9bdc5c0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.ExceptionUtils; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -15,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.time.Duration; import java.time.Period; import java.time.temporal.TemporalAmount; @@ -52,6 +54,19 @@ interface DatetimeArithmeticEvaluator { this.datetimes = datetimes; } + DateTimeArithmeticOperation( + StreamInput in, + OperationSymbol op, + BinaryEvaluator ints, + BinaryEvaluator longs, + BinaryEvaluator ulongs, + BinaryEvaluator doubles, + DatetimeArithmeticEvaluator datetimes + ) throws IOException { + super(in, op, ints, longs, ulongs, doubles); + this.datetimes = datetimes; + } + @Override protected TypeResolution resolveInputType(Expression e, TypeResolutions.ParamOrdinal paramOrdinal) { return TypeResolutions.isType( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java index 375a105f19529..6d84ce3558571 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Div.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible; @@ -15,10 +17,13 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.NumericUtils; +import java.io.IOException; + import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation.OperationSymbol.DIV; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.longToUnsignedLong; public class Div extends EsqlArithmeticOperation implements BinaryComparisonInversible { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Div", Div::new); private DataType type; @@ -35,11 +40,27 @@ public Div(Source source, Expression left, Expression right, DataType type) { DivIntsEvaluator.Factory::new, DivLongsEvaluator.Factory::new, DivUnsignedLongsEvaluator.Factory::new, - (s, lhs, rhs) -> new DivDoublesEvaluator.Factory(source, lhs, rhs) + DivDoublesEvaluator.Factory::new ); this.type = type; } + private Div(StreamInput in) throws IOException { + super( + in, + DIV, + DivIntsEvaluator.Factory::new, + DivLongsEvaluator.Factory::new, + DivUnsignedLongsEvaluator.Factory::new, + DivDoublesEvaluator.Factory::new + ); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public DataType dataType() { if (type == null) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java index 6d63551abd314..7ab6d96181f53 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/EsqlArithmeticOperation.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.compute.operator.EvalOperator.ExpressionEvaluator; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; @@ -17,9 +19,11 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import java.io.IOException; +import java.util.List; import java.util.function.Function; import static org.elasticsearch.common.logging.LoggerMessageFormat.format; @@ -29,6 +33,9 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; public abstract class EsqlArithmeticOperation extends ArithmeticOperation implements EvaluatorMapper { + public static List getNamedWriteables() { + return List.of(Add.ENTRY, Div.ENTRY, Mod.ENTRY, Mul.ENTRY, Sub.ENTRY); + } /** * The only role of this enum is to fit the super constructor that expects a BinaryOperation which is @@ -99,6 +106,26 @@ public interface BinaryEvaluator { this.doubles = doubles; } + EsqlArithmeticOperation( + StreamInput in, + OperationSymbol op, + BinaryEvaluator ints, + BinaryEvaluator longs, + BinaryEvaluator ulongs, + BinaryEvaluator doubles + ) throws IOException { + this( + Source.readFrom((PlanStreamInput) in), + ((PlanStreamInput) in).readExpression(), + ((PlanStreamInput) in).readExpression(), + op, + ints, + longs, + ulongs, + doubles + ); + } + @Override public Object fold() { return EvaluatorMapper.super.fold(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java index bfa10eef9a1c6..151c886bfdf8d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mod.java @@ -7,16 +7,21 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.util.NumericUtils; +import java.io.IOException; + import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation.OperationSymbol.MOD; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.longToUnsignedLong; public class Mod extends EsqlArithmeticOperation { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Mod", Mod::new); public Mod(Source source, Expression left, Expression right) { super( @@ -27,10 +32,26 @@ public Mod(Source source, Expression left, Expression right) { ModIntsEvaluator.Factory::new, ModLongsEvaluator.Factory::new, ModUnsignedLongsEvaluator.Factory::new, - (s, lhs, rhs) -> new ModDoublesEvaluator.Factory(source, lhs, rhs) + ModDoublesEvaluator.Factory::new + ); + } + + private Mod(StreamInput in) throws IOException { + super( + in, + MOD, + ModIntsEvaluator.Factory::new, + ModLongsEvaluator.Factory::new, + ModUnsignedLongsEvaluator.Factory::new, + ModDoublesEvaluator.Factory::new ); } + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, Mod::new, left(), right()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java index efb0b7dbfdc44..08a01fbffcca2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Mul.java @@ -7,16 +7,21 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.BinaryComparisonInversible; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import java.io.IOException; + import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongMultiplyExact; import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation.OperationSymbol.MUL; public class Mul extends EsqlArithmeticOperation implements BinaryComparisonInversible { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Mul", Mul::new); public Mul(Source source, Expression left, Expression right) { super( @@ -27,10 +32,26 @@ public Mul(Source source, Expression left, Expression right) { MulIntsEvaluator.Factory::new, MulLongsEvaluator.Factory::new, MulUnsignedLongsEvaluator.Factory::new, - (s, lhs, rhs) -> new MulDoublesEvaluator.Factory(source, lhs, rhs) + MulDoublesEvaluator.Factory::new + ); + } + + private Mul(StreamInput in) throws IOException { + super( + in, + MUL, + MulIntsEvaluator.Factory::new, + MulLongsEvaluator.Factory::new, + MulUnsignedLongsEvaluator.Factory::new, + MulDoublesEvaluator.Factory::new ); } + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override public ArithmeticOperationFactory binaryComparisonInverse() { return Div::new; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java index b2ae8cff6a697..43398b7750b0d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/Sub.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.compute.ann.Fixed; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -16,6 +18,7 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import java.io.IOException; import java.time.DateTimeException; import java.time.Duration; import java.time.Period; @@ -28,6 +31,7 @@ import static org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation.OperationSymbol.SUB; public class Sub extends DateTimeArithmeticOperation implements BinaryComparisonInversible { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(Expression.class, "Sub", Sub::new); public Sub(Source source, Expression left, Expression right) { super( @@ -38,11 +42,28 @@ public Sub(Source source, Expression left, Expression right) { SubIntsEvaluator.Factory::new, SubLongsEvaluator.Factory::new, SubUnsignedLongsEvaluator.Factory::new, - (s, lhs, rhs) -> new SubDoublesEvaluator.Factory(source, lhs, rhs), + SubDoublesEvaluator.Factory::new, SubDatetimesEvaluator.Factory::new ); } + private Sub(StreamInput in) throws IOException { + super( + in, + SUB, + SubIntsEvaluator.Factory::new, + SubLongsEvaluator.Factory::new, + SubUnsignedLongsEvaluator.Factory::new, + SubDoublesEvaluator.Factory::new, + SubDatetimesEvaluator.Factory::new + ); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected TypeResolution resolveType() { TypeResolution resolution = super.resolveType(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java index e73cf91cd52a8..26a74e7bdb03c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/Equals.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -19,6 +20,12 @@ import java.util.Map; public class Equals extends EsqlBinaryComparison implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "Equals", + EsqlBinaryComparison::readFrom + ); + private static final Map evaluatorMap = Map.ofEntries( Map.entry(DataType.BOOLEAN, EqualsBoolsEvaluator.Factory::new), Map.entry(DataType.INTEGER, EqualsIntsEvaluator.Factory::new), @@ -44,6 +51,11 @@ public Equals(Source source, Expression left, Expression right, ZoneId zoneId) { super(source, left, right, BinaryComparisonOperation.EQ, zoneId, evaluatorMap); } + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, Equals::new, left(), right(), zoneId()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index 41dafecbff76e..a4559e10eaf3a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -21,10 +22,13 @@ import org.elasticsearch.xpack.esql.evaluator.mapper.EvaluatorMapper; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Cast; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import java.io.IOException; import java.time.ZoneId; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -32,6 +36,9 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; public abstract class EsqlBinaryComparison extends BinaryComparison implements EvaluatorMapper { + public static List getNamedWriteables() { + return List.of(Equals.ENTRY, GreaterThan.ENTRY, GreaterThanOrEqual.ENTRY, LessThan.ENTRY, LessThanOrEqual.ENTRY, NotEquals.ENTRY); + } private final Map evaluatorMap; @@ -118,6 +125,26 @@ protected EsqlBinaryComparison( this.functionType = operation; } + public static EsqlBinaryComparison readFrom(StreamInput in) throws IOException { + // TODO this uses a constructor on the operation *and* a name which is confusing. It only needs one. Everything else uses a name. + var source = Source.readFrom((PlanStreamInput) in); + EsqlBinaryComparison.BinaryComparisonOperation operation = EsqlBinaryComparison.BinaryComparisonOperation.readFromStream(in); + var left = ((PlanStreamInput) in).readExpression(); + var right = ((PlanStreamInput) in).readExpression(); + // TODO: Remove zoneId entirely + var zoneId = in.readOptionalZoneId(); + return operation.buildNewInstance(source, left, right); + } + + @Override + public final void writeTo(StreamOutput out) throws IOException { + source().writeTo(out); + functionType.writeTo(out); + ((PlanStreamOutput) out).writeExpression(left()); + ((PlanStreamOutput) out).writeExpression(right()); + out.writeOptionalZoneId(zoneId()); + } + public BinaryComparisonOperation getFunctionType() { return functionType; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java index da639b328b7c2..8ce8bf30ef617 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThan.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -19,6 +20,12 @@ import java.util.Map; public class GreaterThan extends EsqlBinaryComparison implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "GreaterThan", + EsqlBinaryComparison::readFrom + ); + private static final Map evaluatorMap = Map.ofEntries( Map.entry(DataType.INTEGER, GreaterThanIntsEvaluator.Factory::new), Map.entry(DataType.DOUBLE, GreaterThanDoublesEvaluator.Factory::new), @@ -39,6 +46,11 @@ public GreaterThan(Source source, Expression left, Expression right, ZoneId zone super(source, left, right, BinaryComparisonOperation.GT, zoneId, evaluatorMap); } + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, GreaterThan::new, left(), right(), zoneId()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java index 0644cd5df9038..d7bfec75dabfc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqual.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -19,6 +20,12 @@ import java.util.Map; public class GreaterThanOrEqual extends EsqlBinaryComparison implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "GreaterThanOrEqual", + EsqlBinaryComparison::readFrom + ); + private static final Map evaluatorMap = Map.ofEntries( Map.entry(DataType.INTEGER, GreaterThanOrEqualIntsEvaluator.Factory::new), Map.entry(DataType.DOUBLE, GreaterThanOrEqualDoublesEvaluator.Factory::new), @@ -39,6 +46,11 @@ public GreaterThanOrEqual(Source source, Expression left, Expression right, Zone super(source, left, right, BinaryComparisonOperation.GTE, zoneId, evaluatorMap); } + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, GreaterThanOrEqual::new, left(), right(), zoneId()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java index 9302f6e9c5a77..1ce87094e50b3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveBinaryComparison.java @@ -6,17 +6,24 @@ */ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.function.scalar.BinaryScalarFunction; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import java.io.IOException; + public abstract class InsensitiveBinaryComparison extends BinaryScalarFunction { protected InsensitiveBinaryComparison(Source source, Expression left, Expression right) { super(source, left, right); } + protected InsensitiveBinaryComparison(StreamInput in) throws IOException { + super(in); + } + @Override public DataType dataType() { return DataType.BOOLEAN; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java index 5711495dc29eb..c731e44197f2e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEquals.java @@ -9,6 +9,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.compute.ann.Evaluator; @@ -18,12 +20,28 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import java.io.IOException; + public class InsensitiveEquals extends InsensitiveBinaryComparison { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "InsensitiveEquals", + InsensitiveEquals::new + ); public InsensitiveEquals(Source source, Expression left, Expression right) { super(source, left, right); } + private InsensitiveEquals(StreamInput in) throws IOException { + super(in); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, InsensitiveEquals::new, left(), right()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java index 8c6824a9827d0..b1562b6dc2be4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -19,6 +20,11 @@ import java.util.Map; public class LessThan extends EsqlBinaryComparison implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "LessThan", + EsqlBinaryComparison::readFrom + ); private static final Map evaluatorMap = Map.ofEntries( Map.entry(DataType.INTEGER, LessThanIntsEvaluator.Factory::new), @@ -40,6 +46,11 @@ public LessThan(Source source, Expression left, Expression right, ZoneId zoneId) super(source, left, right, BinaryComparisonOperation.LT, zoneId, evaluatorMap); } + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, LessThan::new, left(), right(), zoneId()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java index 0a18c44ea2891..c31e055c8dd1a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqual.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -19,6 +20,12 @@ import java.util.Map; public class LessThanOrEqual extends EsqlBinaryComparison implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "LessThanOrEqual", + EsqlBinaryComparison::readFrom + ); + private static final Map evaluatorMap = Map.ofEntries( Map.entry(DataType.INTEGER, LessThanOrEqualIntsEvaluator.Factory::new), Map.entry(DataType.DOUBLE, LessThanOrEqualDoublesEvaluator.Factory::new), @@ -39,6 +46,11 @@ public LessThanOrEqual(Source source, Expression left, Expression right, ZoneId super(source, left, right, BinaryComparisonOperation.LTE, zoneId, evaluatorMap); } + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Override protected NodeInfo info() { return NodeInfo.create(this, LessThanOrEqual::new, left(), right(), zoneId()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java index 0a60a6da818c1..179ff61d9c017 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEquals.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.compute.ann.Evaluator; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.Negatable; @@ -19,6 +20,12 @@ import java.util.Map; public class NotEquals extends EsqlBinaryComparison implements Negatable { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + Expression.class, + "NotEquals", + EsqlBinaryComparison::readFrom + ); + private static final Map evaluatorMap = Map.ofEntries( Map.entry(DataType.BOOLEAN, NotEqualsBoolsEvaluator.Factory::new), Map.entry(DataType.INTEGER, NotEqualsIntsEvaluator.Factory::new), @@ -44,6 +51,11 @@ public NotEquals(Source source, Expression left, Expression right, ZoneId zoneId super(source, left, right, BinaryComparisonOperation.NEQ, zoneId, evaluatorMap); } + @Override + public String getWriteableName() { + return ENTRY.name; + } + @Evaluator(extraName = "Ints") static boolean processInts(int lhs, int rhs) { return lhs != rhs; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java index 795790949f665..74e8661596e41 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypes.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.io.stream; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.TriFunction; @@ -30,13 +29,13 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Order; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; +import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.BinaryLogic; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; -import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; import org.elasticsearch.xpack.esql.core.expression.predicate.regex.WildcardPattern; @@ -46,7 +45,6 @@ import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; @@ -60,19 +58,12 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.Percentile; import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroid; import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; +import org.elasticsearch.xpack.esql.expression.function.aggregate.TopList; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import org.elasticsearch.xpack.esql.expression.function.grouping.Bucket; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; -import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; -import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Greatest; -import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateDiff; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateExtract; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateFormat; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateParse; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.CIDRMatch; import org.elasticsearch.xpack.esql.expression.function.scalar.ip.IpPrefix; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Atan2; @@ -83,27 +74,11 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.math.Round; import org.elasticsearch.xpack.esql.expression.function.scalar.math.Tau; import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.AbstractMultivalueFunction; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAppend; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvAvg; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvConcat; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvCount; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvDedupe; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvFirst; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvLast; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMax; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMedian; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSlice; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSort; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvSum; -import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvZip; -import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialContains; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialDisjoint; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialIntersects; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialWithin; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.Concat; import org.elasticsearch.xpack.esql.expression.function.scalar.string.EndsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Left; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Locate; @@ -114,23 +89,10 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.string.Split; import org.elasticsearch.xpack.esql.expression.function.scalar.string.StartsWith; import org.elasticsearch.xpack.esql.expression.function.scalar.string.Substring; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToLower; -import org.elasticsearch.xpack.esql.expression.function.scalar.string.ToUpper; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Div; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mod; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Mul; -import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Sub; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; +import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.EsqlArithmeticOperation; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveEquals; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Dissect.Parser; @@ -178,10 +140,6 @@ import java.util.function.Function; import static java.util.Map.entry; -import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT; -import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; -import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; -import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.GEO; import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.Entry.of; import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanReader.readerFromPlanReader; import static org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry.PlanWriter.writerFromPlanWriter; @@ -267,20 +225,6 @@ public static List namedTypeEntries() { of(LogicalPlan.class, OrderBy.class, PlanNamedTypes::writeOrderBy, PlanNamedTypes::readOrderBy), of(LogicalPlan.class, Project.class, PlanNamedTypes::writeProject, PlanNamedTypes::readProject), of(LogicalPlan.class, TopN.class, PlanNamedTypes::writeTopN, PlanNamedTypes::readTopN), - // BinaryComparison - of(EsqlBinaryComparison.class, Equals.class, PlanNamedTypes::writeBinComparison, PlanNamedTypes::readBinComparison), - of(EsqlBinaryComparison.class, NotEquals.class, PlanNamedTypes::writeBinComparison, PlanNamedTypes::readBinComparison), - of(EsqlBinaryComparison.class, GreaterThan.class, PlanNamedTypes::writeBinComparison, PlanNamedTypes::readBinComparison), - of(EsqlBinaryComparison.class, GreaterThanOrEqual.class, PlanNamedTypes::writeBinComparison, PlanNamedTypes::readBinComparison), - of(EsqlBinaryComparison.class, LessThan.class, PlanNamedTypes::writeBinComparison, PlanNamedTypes::readBinComparison), - of(EsqlBinaryComparison.class, LessThanOrEqual.class, PlanNamedTypes::writeBinComparison, PlanNamedTypes::readBinComparison), - // InsensitiveEquals - of( - InsensitiveEquals.class, - InsensitiveEquals.class, - PlanNamedTypes::writeInsensitiveEquals, - PlanNamedTypes::readInsensitiveEquals - ), // InComparison of(ScalarFunction.class, In.class, PlanNamedTypes::writeInComparison, PlanNamedTypes::readInComparison), // RegexMatch @@ -289,27 +233,12 @@ public static List namedTypeEntries() { // BinaryLogic of(BinaryLogic.class, And.class, PlanNamedTypes::writeBinaryLogic, PlanNamedTypes::readBinaryLogic), of(BinaryLogic.class, Or.class, PlanNamedTypes::writeBinaryLogic, PlanNamedTypes::readBinaryLogic), - // UnaryScalarFunction - of(QL_UNARY_SCLR_CLS, IsNotNull.class, PlanNamedTypes::writeQLUnaryScalar, PlanNamedTypes::readQLUnaryScalar), - of(QL_UNARY_SCLR_CLS, IsNull.class, PlanNamedTypes::writeQLUnaryScalar, PlanNamedTypes::readQLUnaryScalar), - of(QL_UNARY_SCLR_CLS, Not.class, PlanNamedTypes::writeQLUnaryScalar, PlanNamedTypes::readQLUnaryScalar), // ScalarFunction of(ScalarFunction.class, Atan2.class, PlanNamedTypes::writeAtan2, PlanNamedTypes::readAtan2), - of(ScalarFunction.class, Case.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), of(ScalarFunction.class, CIDRMatch.class, PlanNamedTypes::writeCIDRMatch, PlanNamedTypes::readCIDRMatch), - of(ScalarFunction.class, Coalesce.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), - of(ScalarFunction.class, Concat.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), - of(ScalarFunction.class, DateDiff.class, PlanNamedTypes::writeDateDiff, PlanNamedTypes::readDateDiff), - of(ScalarFunction.class, DateExtract.class, PlanNamedTypes::writeDateExtract, PlanNamedTypes::readDateExtract), - of(ScalarFunction.class, DateFormat.class, PlanNamedTypes::writeDateFormat, PlanNamedTypes::readDateFormat), - of(ScalarFunction.class, DateParse.class, PlanNamedTypes::writeDateTimeParse, PlanNamedTypes::readDateTimeParse), - of(ScalarFunction.class, DateTrunc.class, PlanNamedTypes::writeDateTrunc, PlanNamedTypes::readDateTrunc), of(ScalarFunction.class, E.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), - of(ScalarFunction.class, Greatest.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), of(ScalarFunction.class, IpPrefix.class, (out, prefix) -> prefix.writeTo(out), IpPrefix::readFrom), - of(ScalarFunction.class, Least.class, PlanNamedTypes::writeVararg, PlanNamedTypes::readVarag), of(ScalarFunction.class, Log.class, PlanNamedTypes::writeLog, PlanNamedTypes::readLog), - of(ScalarFunction.class, Now.class, PlanNamedTypes::writeNow, PlanNamedTypes::readNow), of(ScalarFunction.class, Pi.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), of(ScalarFunction.class, Round.class, PlanNamedTypes::writeRound, PlanNamedTypes::readRound), of(ScalarFunction.class, Pow.class, PlanNamedTypes::writePow, PlanNamedTypes::readPow), @@ -327,14 +256,6 @@ public static List namedTypeEntries() { of(ScalarFunction.class, Split.class, PlanNamedTypes::writeSplit, PlanNamedTypes::readSplit), of(ScalarFunction.class, Tau.class, PlanNamedTypes::writeNoArgScalar, PlanNamedTypes::readNoArgScalar), of(ScalarFunction.class, Replace.class, PlanNamedTypes::writeReplace, PlanNamedTypes::readReplace), - of(ScalarFunction.class, ToLower.class, PlanNamedTypes::writeToLower, PlanNamedTypes::readToLower), - of(ScalarFunction.class, ToUpper.class, PlanNamedTypes::writeToUpper, PlanNamedTypes::readToUpper), - // ArithmeticOperations - of(ArithmeticOperation.class, Add.class, PlanNamedTypes::writeArithmeticOperation, PlanNamedTypes::readArithmeticOperation), - of(ArithmeticOperation.class, Sub.class, PlanNamedTypes::writeArithmeticOperation, PlanNamedTypes::readArithmeticOperation), - of(ArithmeticOperation.class, Mul.class, PlanNamedTypes::writeArithmeticOperation, PlanNamedTypes::readArithmeticOperation), - of(ArithmeticOperation.class, Div.class, PlanNamedTypes::writeArithmeticOperation, PlanNamedTypes::readArithmeticOperation), - of(ArithmeticOperation.class, Mod.class, PlanNamedTypes::writeArithmeticOperation, PlanNamedTypes::readArithmeticOperation), // GroupingFunctions of(GroupingFunction.class, Bucket.class, PlanNamedTypes::writeBucket, PlanNamedTypes::readBucket), // AggregateFunctions @@ -348,36 +269,26 @@ public static List namedTypeEntries() { of(AggregateFunction.class, Percentile.class, PlanNamedTypes::writePercentile, PlanNamedTypes::readPercentile), of(AggregateFunction.class, SpatialCentroid.class, PlanNamedTypes::writeAggFunction, PlanNamedTypes::readAggFunction), of(AggregateFunction.class, Sum.class, PlanNamedTypes::writeAggFunction, PlanNamedTypes::readAggFunction), - of(AggregateFunction.class, Values.class, PlanNamedTypes::writeAggFunction, PlanNamedTypes::readAggFunction), - // Multivalue functions - of(ScalarFunction.class, MvAppend.class, PlanNamedTypes::writeMvAppend, PlanNamedTypes::readMvAppend), - of(ScalarFunction.class, MvAvg.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvCount.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvConcat.class, PlanNamedTypes::writeMvConcat, PlanNamedTypes::readMvConcat), - of(ScalarFunction.class, MvDedupe.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvFirst.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvLast.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvMax.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvMedian.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvMin.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvSort.class, PlanNamedTypes::writeMvSort, PlanNamedTypes::readMvSort), - of(ScalarFunction.class, MvSlice.class, PlanNamedTypes::writeMvSlice, PlanNamedTypes::readMvSlice), - of(ScalarFunction.class, MvSum.class, PlanNamedTypes::writeMvFunction, PlanNamedTypes::readMvFunction), - of(ScalarFunction.class, MvZip.class, PlanNamedTypes::writeMvZip, PlanNamedTypes::readMvZip), - // Expressions (other) - of(Expression.class, Literal.class, PlanNamedTypes::writeLiteral, PlanNamedTypes::readLiteral), - of(Expression.class, Order.class, PlanNamedTypes::writeOrder, PlanNamedTypes::readOrder) + of(AggregateFunction.class, TopList.class, (out, prefix) -> prefix.writeTo(out), TopList::readFrom), + of(AggregateFunction.class, Values.class, PlanNamedTypes::writeAggFunction, PlanNamedTypes::readAggFunction) ); List entries = new ArrayList<>(declared); // From NamedWriteables - for (NamedWriteableRegistry.Entry e : UnaryScalarFunction.getNamedWriteables()) { - entries.add(of(ESQL_UNARY_SCLR_CLS, e)); - } - for (NamedWriteableRegistry.Entry e : NamedExpression.getNamedWriteables()) { - entries.add(of(Expression.class, e)); + for (List ee : List.of( + AbstractMultivalueFunction.getNamedWriteables(), + EsqlArithmeticOperation.getNamedWriteables(), + EsqlBinaryComparison.getNamedWriteables(), + EsqlScalarFunction.getNamedWriteables(), + FullTextPredicate.getNamedWriteables(), + NamedExpression.getNamedWriteables(), + UnaryScalarFunction.getNamedWriteables(), + List.of(UnsupportedAttribute.ENTRY, Literal.ENTRY, org.elasticsearch.xpack.esql.expression.Order.ENTRY) + )) { + for (NamedWriteableRegistry.Entry e : ee) { + entries.add(of(Expression.class, e)); + } } - entries.add(of(Expression.class, UnsupportedAttribute.ENTRY)); return entries; } @@ -678,14 +589,14 @@ static OrderExec readOrderExec(PlanStreamInput in) throws IOException { return new OrderExec( Source.readFrom(in), in.readPhysicalPlanNode(), - in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)) + in.readCollectionAsList(org.elasticsearch.xpack.esql.expression.Order::new) ); } static void writeOrderExec(PlanStreamOutput out, OrderExec orderExec) throws IOException { Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(orderExec.child()); - out.writeCollection(orderExec.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder)); + out.writeCollection(orderExec.order()); } static ProjectExec readProjectExec(PlanStreamInput in) throws IOException { @@ -731,7 +642,7 @@ static TopNExec readTopNExec(PlanStreamInput in) throws IOException { return new TopNExec( Source.readFrom(in), in.readPhysicalPlanNode(), - in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)), + in.readCollectionAsList(org.elasticsearch.xpack.esql.expression.Order::new), in.readNamed(Expression.class), in.readOptionalVInt() ); @@ -740,7 +651,7 @@ static TopNExec readTopNExec(PlanStreamInput in) throws IOException { static void writeTopNExec(PlanStreamOutput out, TopNExec topNExec) throws IOException { Source.EMPTY.writeTo(out); out.writePhysicalPlanNode(topNExec.child()); - out.writeCollection(topNExec.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder)); + out.writeCollection(topNExec.order()); out.writeExpression(topNExec.limit()); out.writeOptionalVInt(topNExec.estimatedRowSize()); } @@ -969,14 +880,14 @@ static OrderBy readOrderBy(PlanStreamInput in) throws IOException { return new OrderBy( Source.readFrom(in), in.readLogicalPlanNode(), - in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)) + in.readCollectionAsList(org.elasticsearch.xpack.esql.expression.Order::new) ); } static void writeOrderBy(PlanStreamOutput out, OrderBy order) throws IOException { Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(order.child()); - out.writeCollection(order.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder)); + out.writeCollection(order.order()); } static Project readProject(PlanStreamInput in) throws IOException { @@ -993,52 +904,18 @@ static TopN readTopN(PlanStreamInput in) throws IOException { return new TopN( Source.readFrom(in), in.readLogicalPlanNode(), - in.readCollectionAsList(readerFromPlanReader(PlanNamedTypes::readOrder)), - in.readNamed(Expression.class) + in.readCollectionAsList(org.elasticsearch.xpack.esql.expression.Order::new), + in.readExpression() ); } static void writeTopN(PlanStreamOutput out, TopN topN) throws IOException { Source.EMPTY.writeTo(out); out.writeLogicalPlanNode(topN.child()); - out.writeCollection(topN.order(), writerFromPlanWriter(PlanNamedTypes::writeOrder)); + out.writeCollection(topN.order()); out.writeExpression(topN.limit()); } - // -- BinaryComparison - - public static EsqlBinaryComparison readBinComparison(PlanStreamInput in, String name) throws IOException { - var source = Source.readFrom(in); - EsqlBinaryComparison.BinaryComparisonOperation operation = EsqlBinaryComparison.BinaryComparisonOperation.readFromStream(in); - var left = in.readExpression(); - var right = in.readExpression(); - // TODO: Remove zoneId entirely - var zoneId = in.readOptionalZoneId(); - return operation.buildNewInstance(source, left, right); - } - - public static void writeBinComparison(PlanStreamOutput out, EsqlBinaryComparison binaryComparison) throws IOException { - binaryComparison.source().writeTo(out); - binaryComparison.getFunctionType().writeTo(out); - out.writeExpression(binaryComparison.left()); - out.writeExpression(binaryComparison.right()); - out.writeOptionalZoneId(binaryComparison.zoneId()); - } - - // -- InsensitiveEquals - static InsensitiveEquals readInsensitiveEquals(PlanStreamInput in, String name) throws IOException { - var source = Source.readFrom(in); - var left = in.readExpression(); - var right = in.readExpression(); - return new InsensitiveEquals(source, left, right); - } - - static void writeInsensitiveEquals(PlanStreamOutput out, InsensitiveEquals eq) throws IOException { - eq.source().writeTo(out); - out.writeExpression(eq.left()); - out.writeExpression(eq.right()); - } - // -- InComparison static In readInComparison(PlanStreamInput in) throws IOException { @@ -1175,32 +1052,6 @@ static void writeBucket(PlanStreamOutput out, Bucket bucket) throws IOException out.writeOptionalExpression(bucket.to()); } - static final Map, ScalarFunction>> VARARG_CTORS = Map.ofEntries( - entry(name(Case.class), Case::new), - entry(name(Coalesce.class), Coalesce::new), - entry(name(Concat.class), Concat::new), - entry(name(Greatest.class), Greatest::new), - entry(name(Least.class), Least::new) - ); - - static ScalarFunction readVarag(PlanStreamInput in, String name) throws IOException { - return VARARG_CTORS.get(name) - .apply( - Source.readFrom(in), - in.readExpression(), - in.readCollectionAsList(readerFromPlanReader(PlanStreamInput::readExpression)) - ); - } - - static void writeVararg(PlanStreamOutput out, ScalarFunction vararg) throws IOException { - vararg.source().writeTo(out); - out.writeExpression(vararg.children().get(0)); - out.writeCollection( - vararg.children().subList(1, vararg.children().size()), - writerFromPlanWriter(PlanStreamOutput::writeExpression) - ); - } - static CountDistinct readCountDistinct(PlanStreamInput in) throws IOException { return new CountDistinct(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } @@ -1213,67 +1064,6 @@ static void writeCountDistinct(PlanStreamOutput out, CountDistinct countDistinct out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); } - static DateDiff readDateDiff(PlanStreamInput in) throws IOException { - return new DateDiff(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readExpression()); - } - - static void writeDateDiff(PlanStreamOutput out, DateDiff function) throws IOException { - Source.EMPTY.writeTo(out); - List fields = function.children(); - assert fields.size() == 3; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - out.writeExpression(fields.get(2)); - } - - static DateExtract readDateExtract(PlanStreamInput in) throws IOException { - return new DateExtract(Source.readFrom(in), in.readExpression(), in.readExpression(), in.configuration()); - } - - static void writeDateExtract(PlanStreamOutput out, DateExtract function) throws IOException { - function.source().writeTo(out); - List fields = function.children(); - assert fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - } - - static DateFormat readDateFormat(PlanStreamInput in) throws IOException { - return new DateFormat(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class), in.configuration()); - } - - static void writeDateFormat(PlanStreamOutput out, DateFormat dateFormat) throws IOException { - dateFormat.source().writeTo(out); - List fields = dateFormat.children(); - assert fields.size() == 1 || fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); - } - - static DateParse readDateTimeParse(PlanStreamInput in) throws IOException { - return new DateParse(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); - } - - static void writeDateTimeParse(PlanStreamOutput out, DateParse function) throws IOException { - function.source().writeTo(out); - List fields = function.children(); - assert fields.size() == 1 || fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); - } - - static DateTrunc readDateTrunc(PlanStreamInput in) throws IOException { - return new DateTrunc(Source.readFrom(in), in.readExpression(), in.readExpression()); - } - - static void writeDateTrunc(PlanStreamOutput out, DateTrunc dateTrunc) throws IOException { - dateTrunc.source().writeTo(out); - List fields = dateTrunc.children(); - assert fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - } - static SpatialIntersects readIntersects(PlanStreamInput in) throws IOException { return new SpatialIntersects(Source.EMPTY, in.readExpression(), in.readExpression()); } @@ -1295,14 +1085,6 @@ static void writeSpatialRelatesFunction(PlanStreamOutput out, SpatialRelatesFunc out.writeExpression(spatialRelatesFunction.right()); } - static Now readNow(PlanStreamInput in) throws IOException { - return new Now(Source.readFrom(in), in.configuration()); - } - - static void writeNow(PlanStreamOutput out, Now function) throws IOException { - Source.EMPTY.writeTo(out); - } - static Round readRound(PlanStreamInput in) throws IOException { return new Round(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); } @@ -1397,22 +1179,6 @@ static void writeReplace(PlanStreamOutput out, Replace replace) throws IOExcepti out.writeExpression(fields.get(2)); } - static ToLower readToLower(PlanStreamInput in) throws IOException { - return new ToLower(Source.EMPTY, in.readExpression(), in.configuration()); - } - - static void writeToLower(PlanStreamOutput out, ToLower toLower) throws IOException { - out.writeExpression(toLower.field()); - } - - static ToUpper readToUpper(PlanStreamInput in) throws IOException { - return new ToUpper(Source.EMPTY, in.readExpression(), in.configuration()); - } - - static void writeToUpper(PlanStreamOutput out, ToUpper toUpper) throws IOException { - out.writeExpression(toUpper.field()); - } - static Left readLeft(PlanStreamInput in) throws IOException { return new Left(Source.readFrom(in), in.readExpression(), in.readExpression()); } @@ -1475,29 +1241,6 @@ static void writeCIDRMatch(PlanStreamOutput out, CIDRMatch cidrMatch) throws IOE out.writeCollection(children.subList(1, children.size()), writerFromPlanWriter(PlanStreamOutput::writeExpression)); } - // -- ArithmeticOperations - - static final Map> ARITHMETIC_CTRS = Map.ofEntries( - entry(name(Add.class), Add::new), - entry(name(Sub.class), Sub::new), - entry(name(Mul.class), Mul::new), - entry(name(Div.class), Div::new), - entry(name(Mod.class), Mod::new) - ); - - static ArithmeticOperation readArithmeticOperation(PlanStreamInput in, String name) throws IOException { - var source = Source.readFrom(in); - var left = in.readExpression(); - var right = in.readExpression(); - return ARITHMETIC_CTRS.get(name).apply(source, left, right); - } - - static void writeArithmeticOperation(PlanStreamOutput out, ArithmeticOperation arithmeticOperation) throws IOException { - arithmeticOperation.source().writeTo(out); - out.writeExpression(arithmeticOperation.left()); - out.writeExpression(arithmeticOperation.right()); - } - // -- Aggregations static final Map> AGG_CTRS = Map.ofEntries( entry(name(Avg.class), Avg::new), @@ -1520,115 +1263,6 @@ static void writeAggFunction(PlanStreamOutput out, AggregateFunction aggregateFu out.writeExpression(aggregateFunction.field()); } - // -- Multivalue functions - static final Map> MV_CTRS = Map.ofEntries( - entry(name(MvAvg.class), MvAvg::new), - entry(name(MvCount.class), MvCount::new), - entry(name(MvDedupe.class), MvDedupe::new), - entry(name(MvFirst.class), MvFirst::new), - entry(name(MvLast.class), MvLast::new), - entry(name(MvMax.class), MvMax::new), - entry(name(MvMedian.class), MvMedian::new), - entry(name(MvMin.class), MvMin::new), - entry(name(MvSum.class), MvSum::new) - ); - - static AbstractMultivalueFunction readMvFunction(PlanStreamInput in, String name) throws IOException { - return MV_CTRS.get(name).apply(Source.readFrom(in), in.readExpression()); - } - - static void writeMvFunction(PlanStreamOutput out, AbstractMultivalueFunction fn) throws IOException { - Source.EMPTY.writeTo(out); - out.writeExpression(fn.field()); - } - - static MvConcat readMvConcat(PlanStreamInput in) throws IOException { - return new MvConcat(Source.readFrom(in), in.readExpression(), in.readExpression()); - } - - static void writeMvConcat(PlanStreamOutput out, MvConcat fn) throws IOException { - Source.EMPTY.writeTo(out); - out.writeExpression(fn.left()); - out.writeExpression(fn.right()); - } - - // -- Expressions (other) - - static Literal readLiteral(PlanStreamInput in) throws IOException { - Source source = Source.readFrom(in); - Object value = in.readGenericValue(); - DataType dataType = DataType.readFrom(in); - return new Literal(source, mapToLiteralValue(in, dataType, value), dataType); - } - - static void writeLiteral(PlanStreamOutput out, Literal literal) throws IOException { - Source.EMPTY.writeTo(out); - out.writeGenericValue(mapFromLiteralValue(out, literal.dataType(), literal.value())); - out.writeString(literal.dataType().typeName()); - } - - /** - * Not all literal values are currently supported in StreamInput/StreamOutput as generic values. - * This mapper allows for addition of new and interesting values without (yet) adding to StreamInput/Output. - * This makes the most sense during the pre-GA version of ESQL. When we get near GA we might want to push this down. - *

    - * For the spatial point type support we need to care about the fact that 8.12.0 uses encoded longs for serializing - * while 8.13 uses WKB. - */ - private static Object mapFromLiteralValue(PlanStreamOutput out, DataType dataType, Object value) { - if (dataType == GEO_POINT || dataType == CARTESIAN_POINT) { - // In 8.12.0 we serialized point literals as encoded longs, but now use WKB - if (out.getTransportVersion().before(TransportVersions.V_8_13_0)) { - if (value instanceof List list) { - return list.stream().map(v -> mapFromLiteralValue(out, dataType, v)).toList(); - } - return wkbAsLong(dataType, (BytesRef) value); - } - } - return value; - } - - /** - * Not all literal values are currently supported in StreamInput/StreamOutput as generic values. - * This mapper allows for addition of new and interesting values without (yet) changing StreamInput/Output. - */ - private static Object mapToLiteralValue(PlanStreamInput in, DataType dataType, Object value) { - if (dataType == GEO_POINT || dataType == CARTESIAN_POINT) { - // In 8.12.0 we serialized point literals as encoded longs, but now use WKB - if (in.getTransportVersion().before(TransportVersions.V_8_13_0)) { - if (value instanceof List list) { - return list.stream().map(v -> mapToLiteralValue(in, dataType, v)).toList(); - } - return longAsWKB(dataType, (Long) value); - } - } - return value; - } - - private static BytesRef longAsWKB(DataType dataType, long encoded) { - return dataType == GEO_POINT ? GEO.longAsWkb(encoded) : CARTESIAN.longAsWkb(encoded); - } - - private static long wkbAsLong(DataType dataType, BytesRef wkb) { - return dataType == GEO_POINT ? GEO.wkbAsLong(wkb) : CARTESIAN.wkbAsLong(wkb); - } - - static Order readOrder(PlanStreamInput in) throws IOException { - return new org.elasticsearch.xpack.esql.expression.Order( - Source.readFrom(in), - in.readNamed(Expression.class), - in.readEnum(Order.OrderDirection.class), - in.readEnum(Order.NullsPosition.class) - ); - } - - static void writeOrder(PlanStreamOutput out, Order order) throws IOException { - Source.EMPTY.writeTo(out); - out.writeExpression(order.child()); - out.writeEnum(order.direction()); - out.writeEnum(order.nullsPosition()); - } - // -- ancillary supporting classes of plan nodes, etc static EsQueryExec.FieldSort readFieldSort(PlanStreamInput in) throws IOException { @@ -1682,54 +1316,4 @@ static void writeLog(PlanStreamOutput out, Log log) throws IOException { out.writeExpression(fields.get(0)); out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); } - - static MvSort readMvSort(PlanStreamInput in) throws IOException { - return new MvSort(Source.readFrom(in), in.readExpression(), in.readOptionalNamed(Expression.class)); - } - - static void writeMvSort(PlanStreamOutput out, MvSort mvSort) throws IOException { - mvSort.source().writeTo(out); - List fields = mvSort.children(); - assert fields.size() == 1 || fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeOptionalWriteable(fields.size() == 2 ? o -> out.writeExpression(fields.get(1)) : null); - } - - static MvSlice readMvSlice(PlanStreamInput in) throws IOException { - return new MvSlice(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); - } - - static void writeMvSlice(PlanStreamOutput out, MvSlice fn) throws IOException { - Source.EMPTY.writeTo(out); - List fields = fn.children(); - assert fields.size() == 2 || fields.size() == 3; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - out.writeOptionalWriteable(fields.size() == 3 ? o -> out.writeExpression(fields.get(2)) : null); - } - - static MvZip readMvZip(PlanStreamInput in) throws IOException { - return new MvZip(Source.readFrom(in), in.readExpression(), in.readExpression(), in.readOptionalNamed(Expression.class)); - } - - static void writeMvZip(PlanStreamOutput out, MvZip fn) throws IOException { - Source.EMPTY.writeTo(out); - List fields = fn.children(); - assert fields.size() == 2 || fields.size() == 3; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - out.writeOptionalWriteable(fields.size() == 3 ? o -> out.writeExpression(fields.get(2)) : null); - } - - static MvAppend readMvAppend(PlanStreamInput in) throws IOException { - return new MvAppend(Source.readFrom(in), in.readExpression(), in.readExpression()); - } - - static void writeMvAppend(PlanStreamOutput out, MvAppend fn) throws IOException { - Source.EMPTY.writeTo(out); - List fields = fn.children(); - assert fields.size() == 2; - out.writeExpression(fields.get(0)); - out.writeExpression(fields.get(1)); - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java index c5f96988a2ed5..384d3a8cea840 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizer.java @@ -34,7 +34,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; -import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.PropagateEmptyRelation; +import org.elasticsearch.xpack.esql.optimizer.rules.PropagateEmptyRelation; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Eval; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java index 4e2cb2c8223e6..aaf9f8e63d795 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizer.java @@ -7,98 +7,78 @@ package org.elasticsearch.xpack.esql.optimizer; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.compute.data.Block; -import org.elasticsearch.compute.data.BlockFactory; -import org.elasticsearch.compute.data.BlockUtils; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.VerificationException; -import org.elasticsearch.xpack.esql.core.analyzer.AnalyzerRules; import org.elasticsearch.xpack.esql.core.common.Failures; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; -import org.elasticsearch.xpack.esql.core.expression.AttributeSet; -import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; import org.elasticsearch.xpack.esql.core.expression.Expression; -import org.elasticsearch.xpack.esql.core.expression.ExpressionSet; import org.elasticsearch.xpack.esql.core.expression.Expressions; -import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; -import org.elasticsearch.xpack.esql.core.expression.Literal; -import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Order; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; -import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; -import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; -import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; -import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern; -import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; -import org.elasticsearch.xpack.esql.core.plan.logical.Filter; -import org.elasticsearch.xpack.esql.core.plan.logical.Limit; import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.core.rule.ParameterizedRule; import org.elasticsearch.xpack.esql.core.rule.ParameterizedRuleExecutor; -import org.elasticsearch.xpack.esql.core.rule.Rule; -import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.util.CollectionUtils; -import org.elasticsearch.xpack.esql.core.util.Holder; -import org.elasticsearch.xpack.esql.core.util.StringUtils; -import org.elasticsearch.xpack.esql.expression.SurrogateExpression; -import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; -import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; -import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; -import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; -import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; -import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; -import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; +import org.elasticsearch.xpack.esql.optimizer.rules.AddDefaultTopN; import org.elasticsearch.xpack.esql.optimizer.rules.BooleanFunctionEqualsElimination; +import org.elasticsearch.xpack.esql.optimizer.rules.BooleanSimplification; import org.elasticsearch.xpack.esql.optimizer.rules.CombineDisjunctionsToIn; +import org.elasticsearch.xpack.esql.optimizer.rules.CombineEvals; +import org.elasticsearch.xpack.esql.optimizer.rules.CombineProjections; import org.elasticsearch.xpack.esql.optimizer.rules.ConstantFolding; +import org.elasticsearch.xpack.esql.optimizer.rules.ConvertStringToByteRef; +import org.elasticsearch.xpack.esql.optimizer.rules.DuplicateLimitAfterMvExpand; +import org.elasticsearch.xpack.esql.optimizer.rules.FoldNull; import org.elasticsearch.xpack.esql.optimizer.rules.LiteralsOnTheRight; +import org.elasticsearch.xpack.esql.optimizer.rules.PartiallyFoldCase; +import org.elasticsearch.xpack.esql.optimizer.rules.PropagateEmptyRelation; import org.elasticsearch.xpack.esql.optimizer.rules.PropagateEquals; +import org.elasticsearch.xpack.esql.optimizer.rules.PropagateEvalFoldables; +import org.elasticsearch.xpack.esql.optimizer.rules.PropagateNullable; +import org.elasticsearch.xpack.esql.optimizer.rules.PruneColumns; +import org.elasticsearch.xpack.esql.optimizer.rules.PruneEmptyPlans; +import org.elasticsearch.xpack.esql.optimizer.rules.PruneFilters; import org.elasticsearch.xpack.esql.optimizer.rules.PruneLiteralsInOrderBy; +import org.elasticsearch.xpack.esql.optimizer.rules.PruneOrderByBeforeStats; +import org.elasticsearch.xpack.esql.optimizer.rules.PruneRedundantSortClauses; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineLimits; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineOrderBy; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownEnrich; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownEval; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownRegexExtract; +import org.elasticsearch.xpack.esql.optimizer.rules.RemoveStatsOverride; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceAliasingEvalWithProject; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceLimitAndSortAsTopN; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceLookupWithJoin; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceOrderByExpressionWithEval; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceRegexMatch; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceStatsAggExpressionWithEval; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceStatsNestedExpressionWithEval; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceTrivialTypeConversions; import org.elasticsearch.xpack.esql.optimizer.rules.SetAsOptimized; import org.elasticsearch.xpack.esql.optimizer.rules.SimplifyComparisonsArithmetics; -import org.elasticsearch.xpack.esql.plan.logical.Aggregate; -import org.elasticsearch.xpack.esql.plan.logical.Enrich; -import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.optimizer.rules.SkipQueryOnEmptyMappings; +import org.elasticsearch.xpack.esql.optimizer.rules.SkipQueryOnLimitZero; +import org.elasticsearch.xpack.esql.optimizer.rules.SplitInWithFoldableValue; +import org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSpatialSurrogates; +import org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSurrogates; import org.elasticsearch.xpack.esql.plan.logical.Eval; -import org.elasticsearch.xpack.esql.plan.logical.Lookup; -import org.elasticsearch.xpack.esql.plan.logical.MvExpand; import org.elasticsearch.xpack.esql.plan.logical.Project; -import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; -import org.elasticsearch.xpack.esql.plan.logical.TopN; -import org.elasticsearch.xpack.esql.plan.logical.join.Join; -import org.elasticsearch.xpack.esql.plan.logical.join.JoinType; import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; -import org.elasticsearch.xpack.esql.planner.PlannerUtils; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.function.Predicate; import static java.util.Arrays.asList; -import static java.util.Collections.singleton; -import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; -import static org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.TransformDirection; -import static org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.TransformDirection.DOWN; import static org.elasticsearch.xpack.esql.expression.NamedExpressions.mergeOutputExpressions; -import static org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.SubstituteSurrogates.rawTemporaryName; public class LogicalPlanOptimizer extends ParameterizedRuleExecutor { @@ -195,807 +175,14 @@ protected static List> rules() { return asList(substitutions(), operators(), skip, cleanup(), defaultTopN, label); } - // TODO: currently this rule only works for aggregate functions (AVG) - static class SubstituteSurrogates extends OptimizerRules.OptimizerRule { - - SubstituteSurrogates() { - super(TransformDirection.UP); - } - - @Override - protected LogicalPlan rule(Aggregate aggregate) { - var aggs = aggregate.aggregates(); - List newAggs = new ArrayList<>(aggs.size()); - // existing aggregate and their respective attributes - Map aggFuncToAttr = new HashMap<>(); - // surrogate functions eval - List transientEval = new ArrayList<>(); - boolean changed = false; - - // first pass to check existing aggregates (to avoid duplication and alias waste) - for (NamedExpression agg : aggs) { - if (Alias.unwrap(agg) instanceof AggregateFunction af) { - if ((af instanceof SurrogateExpression se && se.surrogate() != null) == false) { - aggFuncToAttr.put(af, agg.toAttribute()); - } - } - } - - int[] counter = new int[] { 0 }; - // 0. check list of surrogate expressions - for (NamedExpression agg : aggs) { - Expression e = Alias.unwrap(agg); - if (e instanceof SurrogateExpression sf && sf.surrogate() != null) { - changed = true; - Expression s = sf.surrogate(); - - // if the expression is NOT a 1:1 replacement need to add an eval - if (s instanceof AggregateFunction == false) { - // 1. collect all aggregate functions from the expression - var surrogateWithRefs = s.transformUp(AggregateFunction.class, af -> { - // 2. check if they are already use otherwise add them to the Aggregate with some made-up aliases - // 3. replace them inside the expression using the given alias - var attr = aggFuncToAttr.get(af); - // the agg doesn't exist in the Aggregate, create an alias for it and save its attribute - if (attr == null) { - var temporaryName = temporaryName(af, agg, counter[0]++); - // create a synthetic alias (so it doesn't clash with a user defined name) - var newAlias = new Alias(agg.source(), temporaryName, null, af, null, true); - attr = newAlias.toAttribute(); - aggFuncToAttr.put(af, attr); - newAggs.add(newAlias); - } - return attr; - }); - // 4. move the expression as an eval using the original alias - // copy the original alias id so that other nodes using it down stream (e.g. eval referring to the original agg) - // don't have to updated - var aliased = new Alias(agg.source(), agg.name(), null, surrogateWithRefs, agg.toAttribute().id()); - transientEval.add(aliased); - } - // the replacement is another aggregate function, so replace it in place - else { - newAggs.add((NamedExpression) agg.replaceChildren(Collections.singletonList(s))); - } - } else { - newAggs.add(agg); - } - } - - LogicalPlan plan = aggregate; - if (changed) { - var source = aggregate.source(); - if (newAggs.isEmpty() == false) { - plan = new Aggregate(source, aggregate.child(), aggregate.groupings(), newAggs); - } else { - // All aggs actually have been surrogates for (foldable) expressions, e.g. - // \_Aggregate[[],[AVG([1, 2][INTEGER]) AS s]] - // Replace by a local relation with one row, followed by an eval, e.g. - // \_Eval[[MVAVG([1, 2][INTEGER]) AS s]] - // \_LocalRelation[[{e}#21],[ConstantNullBlock[positions=1]]] - plan = new LocalRelation( - source, - List.of(new EmptyAttribute(source)), - LocalSupplier.of(new Block[] { BlockUtils.constantBlock(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, null, 1) }) - ); - } - // 5. force the initial projection in place - if (transientEval.isEmpty() == false) { - plan = new Eval(source, plan, transientEval); - // project away transient fields and re-enforce the original order using references (not copies) to the original aggs - // this works since the replaced aliases have their nameId copied to avoid having to update all references (which has - // a cascading effect) - plan = new Project(source, plan, Expressions.asAttributes(aggs)); - } - } - - return plan; - } - - static String temporaryName(Expression inner, Expression outer, int suffix) { - String in = toString(inner); - String out = toString(outer); - return rawTemporaryName(in, out, String.valueOf(suffix)); - } - - static String rawTemporaryName(String inner, String outer, String suffix) { - return "$$" + inner + "$" + outer + "$" + suffix; - } - - static int TO_STRING_LIMIT = 16; - - static String toString(Expression ex) { - return ex instanceof AggregateFunction af ? af.functionName() : extractString(ex); - } - - static String extractString(Expression ex) { - return ex instanceof NamedExpression ne ? ne.name() : limitToString(ex.sourceText()).replace(' ', '_'); - } - - static String limitToString(String string) { - return string.length() > 16 ? string.substring(0, TO_STRING_LIMIT - 1) + ">" : string; - } - } - - /** - * Currently this works similarly to SurrogateExpression, leaving the logic inside the expressions, - * so each can decide for itself whether or not to change to a surrogate expression. - * But what is actually being done is similar to LiteralsOnTheRight. We can consider in the future moving - * this in either direction, reducing the number of rules, but for now, - * it's a separate rule to reduce the risk of unintended interactions with other rules. - */ - static class SubstituteSpatialSurrogates extends OptimizerRules.OptimizerExpressionRule { - - SubstituteSpatialSurrogates() { - super(TransformDirection.UP); - } - - @Override - protected SpatialRelatesFunction rule(SpatialRelatesFunction function) { - return function.surrogate(); - } - } - - static class ReplaceOrderByExpressionWithEval extends OptimizerRules.OptimizerRule { - private static int counter = 0; - - @Override - protected LogicalPlan rule(OrderBy orderBy) { - int size = orderBy.order().size(); - List evals = new ArrayList<>(size); - List newOrders = new ArrayList<>(size); - - for (int i = 0; i < size; i++) { - var order = orderBy.order().get(i); - if (order.child() instanceof Attribute == false) { - var name = rawTemporaryName("order_by", String.valueOf(i), String.valueOf(counter++)); - var eval = new Alias(order.child().source(), name, order.child()); - newOrders.add(order.replaceChildren(List.of(eval.toAttribute()))); - evals.add(eval); - } else { - newOrders.add(order); - } - } - if (evals.isEmpty()) { - return orderBy; - } else { - var newOrderBy = new OrderBy(orderBy.source(), new Eval(orderBy.source(), orderBy.child(), evals), newOrders); - return new Project(orderBy.source(), newOrderBy, orderBy.output()); - } - } - } - - static class ConvertStringToByteRef extends OptimizerRules.OptimizerExpressionRule { - - ConvertStringToByteRef() { - super(TransformDirection.UP); - } - - @Override - protected Expression rule(Literal lit) { - Object value = lit.value(); - - if (value == null) { - return lit; - } - if (value instanceof String s) { - return Literal.of(lit, new BytesRef(s)); - } - if (value instanceof List l) { - if (l.isEmpty() || false == l.get(0) instanceof String) { - return lit; - } - List byteRefs = new ArrayList<>(l.size()); - for (Object v : l) { - byteRefs.add(new BytesRef(v.toString())); - } - return Literal.of(lit, byteRefs); - } - return lit; - } - } - - static class CombineProjections extends OptimizerRules.OptimizerRule { - - CombineProjections() { - super(TransformDirection.UP); - } - - @Override - @SuppressWarnings("unchecked") - protected LogicalPlan rule(UnaryPlan plan) { - LogicalPlan child = plan.child(); - - if (plan instanceof Project project) { - if (child instanceof Project p) { - // eliminate lower project but first replace the aliases in the upper one - project = p.withProjections(combineProjections(project.projections(), p.projections())); - child = project.child(); - plan = project; - // don't return the plan since the grandchild (now child) might be an aggregate that could not be folded on the way up - // e.g. stats c = count(x) | project c, c as x | project x - // try to apply the rule again opportunistically as another node might be pushed in (a limit might be pushed in) - } - // check if the projection eliminates certain aggregates - // but be mindful of aliases to existing aggregates that we don't want to duplicate to avoid redundant work - if (child instanceof Aggregate a) { - var aggs = a.aggregates(); - var newAggs = projectAggregations(project.projections(), aggs); - // project can be fully removed - if (newAggs != null) { - var newGroups = replacePrunedAliasesUsedInGroupBy(a.groupings(), aggs, newAggs); - plan = new Aggregate(a.source(), a.child(), newGroups, newAggs); - } - } - return plan; - } - - // Agg with underlying Project (group by on sub-queries) - if (plan instanceof Aggregate a) { - if (child instanceof Project p) { - var groupings = a.groupings(); - List groupingAttrs = new ArrayList<>(a.groupings().size()); - for (Expression grouping : groupings) { - if (grouping instanceof Attribute attribute) { - groupingAttrs.add(attribute); - } else { - // After applying ReplaceStatsNestedExpressionWithEval, groupings can only contain attributes. - throw new EsqlIllegalArgumentException("Expected an Attribute, got {}", grouping); - } - } - plan = new Aggregate( - a.source(), - p.child(), - combineUpperGroupingsAndLowerProjections(groupingAttrs, p.projections()), - combineProjections(a.aggregates(), p.projections()) - ); - } - } - - return plan; - } - - // variant of #combineProjections specialized for project followed by agg due to the rewrite rules applied on aggregations - // this method tries to combine the projections by paying attention to: - // - aggregations that are projected away - remove them - // - aliases in the project that point to aggregates - keep them in place (to avoid duplicating the aggs) - private static List projectAggregations( - List upperProjection, - List lowerAggregations - ) { - AttributeSet seen = new AttributeSet(); - for (NamedExpression upper : upperProjection) { - Expression unwrapped = Alias.unwrap(upper); - // projection contains an inner alias (point to an existing fields inside the projection) - if (seen.contains(unwrapped)) { - return null; - } - seen.add(Expressions.attribute(unwrapped)); - } - - lowerAggregations = combineProjections(upperProjection, lowerAggregations); - - return lowerAggregations; - } - - // normally only the upper projections should survive but since the lower list might have aliases definitions - // that might be reused by the upper one, these need to be replaced. - // for example an alias defined in the lower list might be referred in the upper - without replacing it the alias becomes invalid - private static List combineProjections( - List upper, - List lower - ) { - - // collect named expressions declaration in the lower list - AttributeMap namedExpressions = new AttributeMap<>(); - // while also collecting the alias map for resolving the source (f1 = 1, f2 = f1, etc..) - AttributeMap aliases = new AttributeMap<>(); - for (NamedExpression ne : lower) { - // record the alias - aliases.put(ne.toAttribute(), Alias.unwrap(ne)); - - // record named expression as is - if (ne instanceof Alias as) { - Expression child = as.child(); - namedExpressions.put(ne.toAttribute(), as.replaceChild(aliases.resolve(child, child))); - } - } - List replaced = new ArrayList<>(); - - // replace any matching attribute with a lower alias (if there's a match) - // but clean-up non-top aliases at the end - for (NamedExpression ne : upper) { - NamedExpression replacedExp = (NamedExpression) ne.transformUp(Attribute.class, a -> namedExpressions.resolve(a, a)); - replaced.add((NamedExpression) trimNonTopLevelAliases(replacedExp)); - } - return replaced; - } - - private static List combineUpperGroupingsAndLowerProjections( - List upperGroupings, - List lowerProjections - ) { - // Collect the alias map for resolving the source (f1 = 1, f2 = f1, etc..) - AttributeMap aliases = new AttributeMap<>(); - for (NamedExpression ne : lowerProjections) { - // Projections are just aliases for attributes, so casting is safe. - aliases.put(ne.toAttribute(), (Attribute) Alias.unwrap(ne)); - } - - // Replace any matching attribute directly with the aliased attribute from the projection. - AttributeSet replaced = new AttributeSet(); - for (Attribute attr : upperGroupings) { - // All substitutions happen before; groupings must be attributes at this point. - replaced.add(aliases.resolve(attr, attr)); - } - return new ArrayList<>(replaced); - } - - /** - * Replace grouping alias previously contained in the aggregations that might have been projected away. - */ - private List replacePrunedAliasesUsedInGroupBy( - List groupings, - List oldAggs, - List newAggs - ) { - AttributeMap removedAliases = new AttributeMap<>(); - AttributeSet currentAliases = new AttributeSet(Expressions.asAttributes(newAggs)); - - // record only removed aliases - for (NamedExpression ne : oldAggs) { - if (ne instanceof Alias alias) { - var attr = ne.toAttribute(); - if (currentAliases.contains(attr) == false) { - removedAliases.put(attr, alias.child()); - } - } - } - - if (removedAliases.isEmpty()) { - return groupings; - } - - var newGroupings = new ArrayList(groupings.size()); - for (Expression group : groupings) { - var transformed = group.transformUp(Attribute.class, a -> removedAliases.resolve(a, a)); - if (Expressions.anyMatch(newGroupings, g -> Expressions.equalsAsAttribute(g, transformed)) == false) { - newGroupings.add(transformed); - } - } - - return newGroupings; - } - - public static Expression trimNonTopLevelAliases(Expression e) { - return e instanceof Alias a ? a.replaceChild(trimAliases(a.child())) : trimAliases(e); - } - - private static Expression trimAliases(Expression e) { - return e.transformDown(Alias.class, Alias::child); - } - } - - /** - * Combine multiple Evals into one in order to reduce the number of nodes in a plan. - * TODO: eliminate unnecessary fields inside the eval as well - */ - static class CombineEvals extends OptimizerRules.OptimizerRule { - - CombineEvals() { - super(TransformDirection.UP); - } - - @Override - protected LogicalPlan rule(Eval eval) { - LogicalPlan plan = eval; - if (eval.child() instanceof Eval subEval) { - plan = new Eval(eval.source(), subEval.child(), CollectionUtils.combine(subEval.fields(), eval.fields())); - } - return plan; - } - } - - // - // Replace any reference attribute with its source, if it does not affect the result. - // This avoids ulterior look-ups between attributes and its source across nodes. - // - static class PropagateEvalFoldables extends Rule { - - @Override - public LogicalPlan apply(LogicalPlan plan) { - var collectRefs = new AttributeMap(); - - java.util.function.Function replaceReference = r -> collectRefs.resolve(r, r); - - // collect aliases bottom-up - plan.forEachExpressionUp(Alias.class, a -> { - var c = a.child(); - boolean shouldCollect = c.foldable(); - // try to resolve the expression based on an existing foldables - if (shouldCollect == false) { - c = c.transformUp(ReferenceAttribute.class, replaceReference); - shouldCollect = c.foldable(); - } - if (shouldCollect) { - collectRefs.put(a.toAttribute(), Literal.of(c)); - } - }); - if (collectRefs.isEmpty()) { - return plan; - } - - plan = plan.transformUp(p -> { - // Apply the replacement inside Filter and Eval (which shouldn't make a difference) - // TODO: also allow aggregates once aggs on constants are supported. - // C.f. https://github.com/elastic/elasticsearch/issues/100634 - if (p instanceof Filter || p instanceof Eval) { - p = p.transformExpressionsOnly(ReferenceAttribute.class, replaceReference); - } - return p; - }); - - return plan; - } - } - - static class PushDownAndCombineLimits extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(Limit limit) { - if (limit.child() instanceof Limit childLimit) { - var limitSource = limit.limit(); - var l1 = (int) limitSource.fold(); - var l2 = (int) childLimit.limit().fold(); - return new Limit(limit.source(), Literal.of(limitSource, Math.min(l1, l2)), childLimit.child()); - } else if (limit.child() instanceof UnaryPlan unary) { - if (unary instanceof Eval || unary instanceof Project || unary instanceof RegexExtract || unary instanceof Enrich) { - return unary.replaceChild(limit.replaceChild(unary.child())); - } - // check if there's a 'visible' descendant limit lower than the current one - // and if so, align the current limit since it adds no value - // this applies for cases such as | limit 1 | sort field | limit 10 - else { - Limit descendantLimit = descendantLimit(unary); - if (descendantLimit != null) { - var l1 = (int) limit.limit().fold(); - var l2 = (int) descendantLimit.limit().fold(); - if (l2 <= l1) { - return new Limit(limit.source(), Literal.of(limit.limit(), l2), limit.child()); - } - } - } - } else if (limit.child() instanceof Join join) { - if (join.config().type() == JoinType.LEFT && join.right() instanceof LocalRelation) { - // This is a hash join from something like a lookup. - return join.replaceChildren(limit.replaceChild(join.left()), join.right()); - } - } - return limit; - } - - /** - * Checks the existence of another 'visible' Limit, that exists behind an operation that doesn't produce output more data than - * its input (that is not a relation/source nor aggregation). - * P.S. Typically an aggregation produces less data than the input. - */ - private static Limit descendantLimit(UnaryPlan unary) { - UnaryPlan plan = unary; - while (plan instanceof Aggregate == false) { - if (plan instanceof Limit limit) { - return limit; - } else if (plan instanceof MvExpand) { - // the limit that applies to mv_expand shouldn't be changed - // ie "| limit 1 | mv_expand x | limit 20" where we want that last "limit" to apply on expand results - return null; - } - if (plan.child() instanceof UnaryPlan unaryPlan) { - plan = unaryPlan; - } else { - break; - } - } - return null; - } - } - - static class DuplicateLimitAfterMvExpand extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(Limit limit) { - var child = limit.child(); - var shouldSkip = child instanceof Eval - || child instanceof Project - || child instanceof RegexExtract - || child instanceof Enrich - || child instanceof Limit; - - if (shouldSkip == false && child instanceof UnaryPlan unary) { - MvExpand mvExpand = descendantMvExpand(unary); - if (mvExpand != null) { - Limit limitBeforeMvExpand = limitBeforeMvExpand(mvExpand); - // if there is no "appropriate" limit before mv_expand, then push down a copy of the one after it so that: - // - a possible TopN is properly built as low as possible in the tree (closed to Lucene) - // - the input of mv_expand is as small as possible before it is expanded (less rows to inflate and occupy memory) - if (limitBeforeMvExpand == null) { - var duplicateLimit = new Limit(limit.source(), limit.limit(), mvExpand.child()); - return limit.replaceChild(propagateDuplicateLimitUntilMvExpand(duplicateLimit, mvExpand, unary)); - } - } - } - return limit; - } - - private static MvExpand descendantMvExpand(UnaryPlan unary) { - UnaryPlan plan = unary; - AttributeSet filterReferences = new AttributeSet(); - while (plan instanceof Aggregate == false) { - if (plan instanceof MvExpand mve) { - // don't return the mv_expand that has a filter after it which uses the expanded values - // since this will trigger the use of a potentially incorrect (too restrictive) limit further down in the tree - if (filterReferences.isEmpty() == false) { - if (filterReferences.contains(mve.target()) // the same field or reference attribute is used in mv_expand AND filter - || mve.target() instanceof ReferenceAttribute // or the mv_expand attr hasn't yet been resolved to a field attr - // or not all filter references have been resolved to field attributes - || filterReferences.stream().anyMatch(ref -> ref instanceof ReferenceAttribute)) { - return null; - } - } - return mve; - } else if (plan instanceof Filter filter) { - // gather all the filters' references to be checked later when a mv_expand is found - filterReferences.addAll(filter.references()); - } else if (plan instanceof OrderBy) { - // ordering after mv_expand COULD break the order of the results, so the limit shouldn't be copied past mv_expand - // something like from test | sort emp_no | mv_expand job_positions | sort first_name | limit 5 - // (the sort first_name likely changes the order of the docs after sort emp_no, so "limit 5" shouldn't be copied down - return null; - } - - if (plan.child() instanceof UnaryPlan unaryPlan) { - plan = unaryPlan; - } else { - break; - } - } - return null; - } - - private static Limit limitBeforeMvExpand(MvExpand mvExpand) { - UnaryPlan plan = mvExpand; - while (plan instanceof Aggregate == false) { - if (plan instanceof Limit limit) { - return limit; - } - if (plan.child() instanceof UnaryPlan unaryPlan) { - plan = unaryPlan; - } else { - break; - } - } - return null; - } - - private LogicalPlan propagateDuplicateLimitUntilMvExpand(Limit duplicateLimit, MvExpand mvExpand, UnaryPlan child) { - if (child == mvExpand) { - return mvExpand.replaceChild(duplicateLimit); - } else { - return child.replaceChild(propagateDuplicateLimitUntilMvExpand(duplicateLimit, mvExpand, (UnaryPlan) child.child())); - } - } - } - - // 3 in (field, 4, 5) --> 3 in (field) or 3 in (4, 5) - public static class SplitInWithFoldableValue extends OptimizerRules.OptimizerExpressionRule { - - SplitInWithFoldableValue() { - super(TransformDirection.UP); - } - - @Override - protected Expression rule(In in) { - if (in.value().foldable()) { - List foldables = new ArrayList<>(in.list().size()); - List nonFoldables = new ArrayList<>(in.list().size()); - in.list().forEach(e -> { - if (e.foldable() && Expressions.isNull(e) == false) { // keep `null`s, needed for the 3VL - foldables.add(e); - } else { - nonFoldables.add(e); - } - }); - if (foldables.size() > 0 && nonFoldables.size() > 0) { - In withFoldables = new In(in.source(), in.value(), foldables); - In withoutFoldables = new In(in.source(), in.value(), nonFoldables); - return new Or(in.source(), withFoldables, withoutFoldables); - } - } - return in; - } - } - - private static class BooleanSimplification extends org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification { - - BooleanSimplification() { - super(); - } - - @Override - protected Expression maybeSimplifyNegatable(Expression e) { - return null; - } - - } - - static class PruneFilters extends OptimizerRules.PruneFilters { - - @Override - protected LogicalPlan skipPlan(Filter filter) { - return LogicalPlanOptimizer.skipPlan(filter); - } - } - - static class SkipQueryOnLimitZero extends OptimizerRules.SkipQueryOnLimitZero { - - @Override - protected LogicalPlan skipPlan(Limit limit) { - return LogicalPlanOptimizer.skipPlan(limit); - } - } - - static class PruneEmptyPlans extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(UnaryPlan plan) { - return plan.output().isEmpty() ? skipPlan(plan) : plan; - } - } - - static class SkipQueryOnEmptyMappings extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(EsRelation plan) { - return plan.index().concreteIndices().isEmpty() ? new LocalRelation(plan.source(), plan.output(), LocalSupplier.EMPTY) : plan; - } - } - - @SuppressWarnings("removal") - static class PropagateEmptyRelation extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(UnaryPlan plan) { - LogicalPlan p = plan; - if (plan.child() instanceof LocalRelation local && local.supplier() == LocalSupplier.EMPTY) { - // only care about non-grouped aggs might return something (count) - if (plan instanceof Aggregate agg && agg.groupings().isEmpty()) { - List emptyBlocks = aggsFromEmpty(agg.aggregates()); - p = skipPlan(plan, LocalSupplier.of(emptyBlocks.toArray(Block[]::new))); - } else { - p = skipPlan(plan); - } - } - return p; - } - - private List aggsFromEmpty(List aggs) { - List blocks = new ArrayList<>(); - var blockFactory = PlannerUtils.NON_BREAKING_BLOCK_FACTORY; - int i = 0; - for (var agg : aggs) { - // there needs to be an alias - if (Alias.unwrap(agg) instanceof AggregateFunction aggFunc) { - aggOutput(agg, aggFunc, blockFactory, blocks); - } else { - throw new EsqlIllegalArgumentException("Did not expect a non-aliased aggregation {}", agg); - } - } - return blocks; - } - - /** - * The folded aggregation output - this variant is for the coordinator/final. - */ - protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFactory blockFactory, List blocks) { - // look for count(literal) with literal != null - Object value = aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L : null; - var wrapper = BlockUtils.wrapperFor(blockFactory, PlannerUtils.toElementType(aggFunc.dataType()), 1); - wrapper.accept(value); - blocks.add(wrapper.builder().build()); - } - } - - private static LogicalPlan skipPlan(UnaryPlan plan) { + public static LogicalPlan skipPlan(UnaryPlan plan) { return new LocalRelation(plan.source(), plan.output(), LocalSupplier.EMPTY); } - private static LogicalPlan skipPlan(UnaryPlan plan, LocalSupplier supplier) { + public static LogicalPlan skipPlan(UnaryPlan plan, LocalSupplier supplier) { return new LocalRelation(plan.source(), plan.output(), supplier); } - protected static class PushDownAndCombineFilters extends OptimizerRules.OptimizerRule { - @Override - protected LogicalPlan rule(Filter filter) { - LogicalPlan plan = filter; - LogicalPlan child = filter.child(); - Expression condition = filter.condition(); - - if (child instanceof Filter f) { - // combine nodes into a single Filter with updated ANDed condition - plan = f.with(Predicates.combineAnd(List.of(f.condition(), condition))); - } else if (child instanceof Aggregate agg) { // TODO: re-evaluate along with multi-value support - // Only push [parts of] a filter past an agg if these/it operates on agg's grouping[s], not output. - plan = maybePushDownPastUnary( - filter, - agg, - e -> e instanceof Attribute && agg.output().contains(e) && agg.groupings().contains(e) == false - || e instanceof AggregateFunction - ); - } else if (child instanceof Eval eval) { - // Don't push if Filter (still) contains references of Eval's fields. - var attributes = new AttributeSet(Expressions.asAttributes(eval.fields())); - plan = maybePushDownPastUnary(filter, eval, attributes::contains); - } else if (child instanceof RegexExtract re) { - // Push down filters that do not rely on attributes created by RegexExtract - var attributes = new AttributeSet(Expressions.asAttributes(re.extractedFields())); - plan = maybePushDownPastUnary(filter, re, attributes::contains); - } else if (child instanceof Enrich enrich) { - // Push down filters that do not rely on attributes created by Enrich - var attributes = new AttributeSet(Expressions.asAttributes(enrich.enrichFields())); - plan = maybePushDownPastUnary(filter, enrich, attributes::contains); - } else if (child instanceof Project) { - return pushDownPastProject(filter); - } else if (child instanceof OrderBy orderBy) { - // swap the filter with its child - plan = orderBy.replaceChild(filter.with(orderBy.child(), condition)); - } - // cannot push past a Limit, this could change the tailing result set returned - return plan; - } - - private static LogicalPlan maybePushDownPastUnary(Filter filter, UnaryPlan unary, Predicate cannotPush) { - LogicalPlan plan; - List pushable = new ArrayList<>(); - List nonPushable = new ArrayList<>(); - for (Expression exp : Predicates.splitAnd(filter.condition())) { - (exp.anyMatch(cannotPush) ? nonPushable : pushable).add(exp); - } - // Push the filter down even if it might not be pushable all the way to ES eventually: eval'ing it closer to the source, - // potentially still in the Exec Engine, distributes the computation. - if (pushable.size() > 0) { - if (nonPushable.size() > 0) { - Filter pushed = new Filter(filter.source(), unary.child(), Predicates.combineAnd(pushable)); - plan = filter.with(unary.replaceChild(pushed), Predicates.combineAnd(nonPushable)); - } else { - plan = unary.replaceChild(filter.with(unary.child(), filter.condition())); - } - } else { - plan = filter; - } - return plan; - } - } - - protected static class PushDownEval extends OptimizerRules.OptimizerRule { - @Override - protected LogicalPlan rule(Eval eval) { - return pushGeneratingPlanPastProjectAndOrderBy(eval, asAttributes(eval.fields())); - } - } - - protected static class PushDownRegexExtract extends OptimizerRules.OptimizerRule { - @Override - protected LogicalPlan rule(RegexExtract re) { - return pushGeneratingPlanPastProjectAndOrderBy(re, re.extractedFields()); - } - } - - protected static class PushDownEnrich extends OptimizerRules.OptimizerRule { - @Override - protected LogicalPlan rule(Enrich en) { - return pushGeneratingPlanPastProjectAndOrderBy(en, asAttributes(en.enrichFields())); - } - } - /** * Pushes LogicalPlans which generate new attributes (Eval, Grok/Dissect, Enrich), past OrderBys and Projections. * Although it seems arbitrary whether the OrderBy or the Eval is executed first, this transformation ensures that OrderBys only @@ -1022,7 +209,7 @@ protected LogicalPlan rule(Enrich en) { * * ... | eval $$a = a | eval a = b + 1 | sort $$a | drop $$a */ - private static LogicalPlan pushGeneratingPlanPastProjectAndOrderBy(UnaryPlan generatingPlan, List generatedAttributes) { + public static LogicalPlan pushGeneratingPlanPastProjectAndOrderBy(UnaryPlan generatingPlan, List generatedAttributes) { LogicalPlan child = generatingPlan.child(); if (child instanceof OrderBy orderBy) { @@ -1087,164 +274,7 @@ private static AttributeReplacement renameAttributesInExpressions( return new AttributeReplacement(rewrittenExpressions, aliasesForReplacedAttributes); } - protected static class PushDownAndCombineOrderBy extends OptimizerRules.OptimizerRule { - @Override - protected LogicalPlan rule(OrderBy orderBy) { - LogicalPlan child = orderBy.child(); - - if (child instanceof OrderBy childOrder) { - // combine orders - return new OrderBy(orderBy.source(), childOrder.child(), orderBy.order()); - } else if (child instanceof Project) { - return pushDownPastProject(orderBy); - } - - return orderBy; - } - } - - /** - * Remove unused columns created in the plan, in fields inside eval or aggregations inside stats. - */ - static class PruneColumns extends Rule { - - @Override - public LogicalPlan apply(LogicalPlan plan) { - var used = new AttributeSet(); - // don't remove Evals without any Project/Aggregate (which might not occur as the last node in the plan) - var seenProjection = new Holder<>(Boolean.FALSE); - - // start top-to-bottom - // and track used references - var pl = plan.transformDown(p -> { - // skip nodes that simply pass the input through - if (p instanceof Limit) { - return p; - } - - // remember used - boolean recheck; - // analyze the unused items against dedicated 'producer' nodes such as Eval and Aggregate - // perform a loop to retry checking if the current node is completely eliminated - do { - recheck = false; - if (p instanceof Aggregate aggregate) { - var remaining = seenProjection.get() ? removeUnused(aggregate.aggregates(), used) : null; - - if (remaining != null) { - if (remaining.isEmpty()) { - // We still need to have a plan that produces 1 row per group. - if (aggregate.groupings().isEmpty()) { - p = new LocalRelation( - aggregate.source(), - List.of(new EmptyAttribute(aggregate.source())), - LocalSupplier.of( - new Block[] { BlockUtils.constantBlock(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, null, 1) } - ) - ); - } else { - // Aggs cannot produce pages with 0 columns, so retain one grouping. - remaining = List.of(Expressions.attribute(aggregate.groupings().get(0))); - p = new Aggregate(aggregate.source(), aggregate.child(), aggregate.groupings(), remaining); - } - } else { - p = new Aggregate(aggregate.source(), aggregate.child(), aggregate.groupings(), remaining); - } - } - - seenProjection.set(Boolean.TRUE); - } else if (p instanceof Eval eval) { - var remaining = seenProjection.get() ? removeUnused(eval.fields(), used) : null; - // no fields, no eval - if (remaining != null) { - if (remaining.isEmpty()) { - p = eval.child(); - recheck = true; - } else { - p = new Eval(eval.source(), eval.child(), remaining); - } - } - } else if (p instanceof Project) { - seenProjection.set(Boolean.TRUE); - } - } while (recheck); - - used.addAll(p.references()); - - // preserve the state before going to the next node - return p; - }); - - return pl; - } - - /** - * Prunes attributes from the list not found in the given set. - * Returns null if no changed occurred. - */ - private static List removeUnused(List named, AttributeSet used) { - var clone = new ArrayList<>(named); - var it = clone.listIterator(clone.size()); - - // due to Eval, go in reverse - while (it.hasPrevious()) { - N prev = it.previous(); - if (used.contains(prev.toAttribute()) == false) { - it.remove(); - } else { - used.addAll(prev.references()); - } - } - return clone.size() != named.size() ? clone : null; - } - } - - static class PruneOrderByBeforeStats extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(Aggregate agg) { - OrderBy order = findPullableOrderBy(agg.child()); - - LogicalPlan p = agg; - if (order != null) { - p = agg.transformDown(OrderBy.class, o -> o == order ? order.child() : o); - } - return p; - } - - private static OrderBy findPullableOrderBy(LogicalPlan plan) { - OrderBy pullable = null; - if (plan instanceof OrderBy o) { - pullable = o; - } else if (plan instanceof Eval - || plan instanceof Filter - || plan instanceof Project - || plan instanceof RegexExtract - || plan instanceof Enrich) { - pullable = findPullableOrderBy(((UnaryPlan) plan).child()); - } - return pullable; - } - - } - - static class PruneRedundantSortClauses extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(OrderBy plan) { - var referencedAttributes = new ExpressionSet(); - var order = new ArrayList(); - for (Order o : plan.order()) { - if (referencedAttributes.add(o)) { - order.add(o); - } - } - - return plan.order().size() == order.size() ? plan : new OrderBy(plan.source(), plan.child(), order); - } - } - - private static Project pushDownPastProject(UnaryPlan parent) { + public static Project pushDownPastProject(UnaryPlan parent) { if (parent.child() instanceof Project project) { AttributeMap.Builder aliasBuilder = AttributeMap.builder(); project.forEachExpression(Alias.class, a -> aliasBuilder.put(a.toAttribute(), a.child())); @@ -1261,481 +291,7 @@ private static Project pushDownPastProject(UnaryPlan parent) { } } - static class ReplaceLimitAndSortAsTopN extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(Limit plan) { - LogicalPlan p = plan; - if (plan.child() instanceof OrderBy o) { - p = new TopN(plan.source(), o.child(), o.order(), plan.limit()); - } - return p; - } - } - - private static class ReplaceLookupWithJoin extends OptimizerRules.OptimizerRule { - - ReplaceLookupWithJoin() { - super(TransformDirection.UP); - } - - @Override - protected LogicalPlan rule(Lookup lookup) { - // left join between the main relation and the local, lookup relation - return new Join(lookup.source(), lookup.child(), lookup.localRelation(), lookup.joinConfig()); - } - } - - /** - * This adds an explicit TopN node to a plan that only has an OrderBy right before Lucene. - * To date, the only known use case that "needs" this is a query of the form - * from test - * | sort emp_no - * | mv_expand first_name - * | rename first_name AS x - * | where x LIKE "*a*" - * | limit 15 - * - * or - * - * from test - * | sort emp_no - * | mv_expand first_name - * | sort first_name - * | limit 15 - * - * PushDownAndCombineLimits rule will copy the "limit 15" after "sort emp_no" if there is no filter on the expanded values - * OR if there is no sort between "limit" and "mv_expand". - * But, since this type of query has such a filter, the "sort emp_no" will have no limit when it reaches the current rule. - */ - static class AddDefaultTopN extends ParameterizedOptimizerRule { - - @Override - protected LogicalPlan rule(LogicalPlan plan, LogicalOptimizerContext context) { - if (plan instanceof UnaryPlan unary && unary.child() instanceof OrderBy order && order.child() instanceof EsRelation relation) { - var limit = new Literal(plan.source(), context.configuration().resultTruncationMaxSize(), DataType.INTEGER); - return unary.replaceChild(new TopN(plan.source(), relation, order.order(), limit)); - } - return plan; - } - } - - public static class ReplaceRegexMatch extends org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.OptimizerExpressionRule< - RegexMatch> { - - ReplaceRegexMatch() { - super(org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.TransformDirection.DOWN); - } - - @Override - public Expression rule(RegexMatch regexMatch) { - Expression e = regexMatch; - StringPattern pattern = regexMatch.pattern(); - if (pattern.matchesAll()) { - e = new IsNotNull(e.source(), regexMatch.field()); - } else { - String match = pattern.exactMatch(); - if (match != null) { - Literal literal = new Literal(regexMatch.source(), match, DataType.KEYWORD); - e = regexToEquals(regexMatch, literal); - } - } - return e; - } - - protected Expression regexToEquals(RegexMatch regexMatch, Literal literal) { - return new Equals(regexMatch.source(), regexMatch.field(), literal); - } - } - - /** - * Replace nested expressions inside an aggregate with synthetic eval (which end up being projected away by the aggregate). - * stats sum(a + 1) by x % 2 - * becomes - * eval `a + 1` = a + 1, `x % 2` = x % 2 | stats sum(`a+1`_ref) by `x % 2`_ref - */ - static class ReplaceStatsNestedExpressionWithEval extends OptimizerRules.OptimizerRule { - - @Override - protected LogicalPlan rule(Aggregate aggregate) { - List evals = new ArrayList<>(); - Map evalNames = new HashMap<>(); - Map groupingAttributes = new HashMap<>(); - List newGroupings = new ArrayList<>(aggregate.groupings()); - boolean groupingChanged = false; - - // start with the groupings since the aggs might duplicate it - for (int i = 0, s = newGroupings.size(); i < s; i++) { - Expression g = newGroupings.get(i); - // move the alias into an eval and replace it with its attribute - if (g instanceof Alias as) { - groupingChanged = true; - var attr = as.toAttribute(); - evals.add(as); - evalNames.put(as.name(), attr); - newGroupings.set(i, attr); - if (as.child() instanceof GroupingFunction gf) { - groupingAttributes.put(gf, attr); - } - } - } - - Holder aggsChanged = new Holder<>(false); - List aggs = aggregate.aggregates(); - List newAggs = new ArrayList<>(aggs.size()); - - // map to track common expressions - Map expToAttribute = new HashMap<>(); - for (Alias a : evals) { - expToAttribute.put(a.child().canonical(), a.toAttribute()); - } - - int[] counter = new int[] { 0 }; - // for the aggs make sure to unwrap the agg function and check the existing groupings - for (NamedExpression agg : aggs) { - NamedExpression a = (NamedExpression) agg.transformDown(Alias.class, as -> { - // if the child is a nested expression - Expression child = as.child(); - - // shortcut for common scenario - if (child instanceof AggregateFunction af && af.field() instanceof Attribute) { - return as; - } - - // check if the alias matches any from grouping otherwise unwrap it - Attribute ref = evalNames.get(as.name()); - if (ref != null) { - aggsChanged.set(true); - return ref; - } - - // 1. look for the aggregate function - var replaced = child.transformUp(AggregateFunction.class, af -> { - Expression result = af; - - Expression field = af.field(); - // 2. if the field is a nested expression (not attribute or literal), replace it - if (field instanceof Attribute == false && field.foldable() == false) { - // 3. create a new alias if one doesn't exist yet no reference - Attribute attr = expToAttribute.computeIfAbsent(field.canonical(), k -> { - Alias newAlias = new Alias(k.source(), syntheticName(k, af, counter[0]++), null, k, null, true); - evals.add(newAlias); - return newAlias.toAttribute(); - }); - aggsChanged.set(true); - // replace field with attribute - List newChildren = new ArrayList<>(af.children()); - newChildren.set(0, attr); - result = af.replaceChildren(newChildren); - } - return result; - }); - // replace any grouping functions with their references pointing to the added synthetic eval - replaced = replaced.transformDown(GroupingFunction.class, gf -> { - aggsChanged.set(true); - // should never return null, as it's verified. - // but even if broken, the transform will fail safely; otoh, returning `gf` will fail later due to incorrect plan. - return groupingAttributes.get(gf); - }); - - return as.replaceChild(replaced); - }); - - newAggs.add(a); - } - - if (evals.size() > 0) { - var groupings = groupingChanged ? newGroupings : aggregate.groupings(); - var aggregates = aggsChanged.get() ? newAggs : aggregate.aggregates(); - - var newEval = new Eval(aggregate.source(), aggregate.child(), evals); - aggregate = new Aggregate(aggregate.source(), newEval, groupings, aggregates); - } - - return aggregate; - } - - static String syntheticName(Expression expression, AggregateFunction af, int counter) { - return SubstituteSurrogates.temporaryName(expression, af, counter); - } - } - - /** - * Replace nested expressions over aggregates with synthetic eval post the aggregation - * stats a = sum(a) + min(b) by x - * becomes - * stats a1 = sum(a), a2 = min(b) by x | eval a = a1 + a2 | keep a, x - * The rule also considers expressions applied over groups: - * stats a = x + 1 by x becomes stats by x | eval a = x + 1 | keep a, x - * And to combine the two: - * stats a = x + count(*) by x - * becomes - * stats a1 = count(*) by x | eval a = x + a1 | keep a1, x - * Since the logic is very similar, this rule also handles duplicate aggregate functions to avoid duplicate compute - * stats a = min(x), b = min(x), c = count(*), d = count() by g - * becomes - * stats a = min(x), c = count(*) by g | eval b = a, d = c | keep a, b, c, d, g - */ - static class ReplaceStatsAggExpressionWithEval extends OptimizerRules.OptimizerRule { - ReplaceStatsAggExpressionWithEval() { - super(TransformDirection.UP); - } - - @Override - protected LogicalPlan rule(Aggregate aggregate) { - // build alias map - AttributeMap aliases = new AttributeMap<>(); - aggregate.forEachExpressionUp(Alias.class, a -> aliases.put(a.toAttribute(), a.child())); - - // break down each aggregate into AggregateFunction and/or grouping key - // preserve the projection at the end - List aggs = aggregate.aggregates(); - - // root/naked aggs - Map rootAggs = Maps.newLinkedHashMapWithExpectedSize(aggs.size()); - // evals (original expression relying on multiple aggs) - List newEvals = new ArrayList<>(); - List newProjections = new ArrayList<>(); - // track the aggregate aggs (including grouping which is not an AggregateFunction) - List newAggs = new ArrayList<>(); - - Holder changed = new Holder<>(false); - int[] counter = new int[] { 0 }; - - for (NamedExpression agg : aggs) { - if (agg instanceof Alias as) { - // if the child a nested expression - Expression child = as.child(); - - // common case - handle duplicates - if (child instanceof AggregateFunction af) { - AggregateFunction canonical = (AggregateFunction) af.canonical(); - Expression field = canonical.field().transformUp(e -> aliases.resolve(e, e)); - canonical = (AggregateFunction) canonical.replaceChildren( - CollectionUtils.combine(singleton(field), canonical.parameters()) - ); - - Alias found = rootAggs.get(canonical); - // aggregate is new - if (found == null) { - rootAggs.put(canonical, as); - newAggs.add(as); - newProjections.add(as.toAttribute()); - } - // agg already exists - preserve the current alias but point it to the existing agg - // thus don't add it to the list of aggs as we don't want duplicated compute - else { - changed.set(true); - newProjections.add(as.replaceChild(found.toAttribute())); - } - } - // nested expression over aggregate function or groups - // replace them with reference and move the expression into a follow-up eval - else { - changed.set(true); - Expression aggExpression = child.transformUp(AggregateFunction.class, af -> { - AggregateFunction canonical = (AggregateFunction) af.canonical(); - Alias alias = rootAggs.get(canonical); - if (alias == null) { - // create synthetic alias ove the found agg function - alias = new Alias( - af.source(), - syntheticName(canonical, child, counter[0]++), - as.qualifier(), - canonical, - null, - true - ); - // and remember it to remove duplicates - rootAggs.put(canonical, alias); - // add it to the list of aggregates and continue - newAggs.add(alias); - } - // (even when found) return a reference to it - return alias.toAttribute(); - }); - - Alias alias = as.replaceChild(aggExpression); - newEvals.add(alias); - newProjections.add(alias.toAttribute()); - } - } - // not an alias (e.g. grouping field) - else { - newAggs.add(agg); - newProjections.add(agg.toAttribute()); - } - } - - LogicalPlan plan = aggregate; - if (changed.get()) { - Source source = aggregate.source(); - plan = new Aggregate(source, aggregate.child(), aggregate.groupings(), newAggs); - if (newEvals.size() > 0) { - plan = new Eval(source, plan, newEvals); - } - // preserve initial projection - plan = new Project(source, plan, newProjections); - } - - return plan; - } - - static String syntheticName(Expression expression, Expression af, int counter) { - return SubstituteSurrogates.temporaryName(expression, af, counter); - } - } - - /** - * Replace aliasing evals (eval x=a) with a projection which can be further combined / simplified. - * The rule gets applied only if there's another project (Project/Stats) above it. - * - * Needs to take into account shadowing of potentially intermediate fields: - * eval x = a + 1, y = x, z = y + 1, y = z, w = y + 1 - * The output should be - * eval x = a + 1, z = a + 1 + 1, w = a + 1 + 1 - * project x, z, z as y, w - */ - static class ReplaceAliasingEvalWithProject extends Rule { - - @Override - public LogicalPlan apply(LogicalPlan logicalPlan) { - Holder enabled = new Holder<>(false); - - return logicalPlan.transformDown(p -> { - // found projection, turn enable flag on - if (p instanceof Aggregate || p instanceof Project) { - enabled.set(true); - } else if (enabled.get() && p instanceof Eval eval) { - p = rule(eval); - } - - return p; - }); - } - - private LogicalPlan rule(Eval eval) { - LogicalPlan plan = eval; - - // holds simple aliases such as b = a, c = b, d = c - AttributeMap basicAliases = new AttributeMap<>(); - // same as above but keeps the original expression - AttributeMap basicAliasSources = new AttributeMap<>(); - - List keptFields = new ArrayList<>(); - - var fields = eval.fields(); - for (int i = 0, size = fields.size(); i < size; i++) { - Alias field = fields.get(i); - Expression child = field.child(); - var attribute = field.toAttribute(); - // put the aliases in a separate map to separate the underlying resolve from other aliases - if (child instanceof Attribute) { - basicAliases.put(attribute, child); - basicAliasSources.put(attribute, field); - } else { - // be lazy and start replacing name aliases only if needed - if (basicAliases.size() > 0) { - // update the child through the field - field = (Alias) field.transformUp(e -> basicAliases.resolve(e, e)); - } - keptFields.add(field); - } - } - - // at least one alias encountered, move it into a project - if (basicAliases.size() > 0) { - // preserve the eval output (takes care of shadowing and order) but replace the basic aliases - List projections = new ArrayList<>(eval.output()); - // replace the removed aliases with their initial definition - however use the output to preserve the shadowing - for (int i = projections.size() - 1; i >= 0; i--) { - NamedExpression project = projections.get(i); - projections.set(i, basicAliasSources.getOrDefault(project, project)); - } - - LogicalPlan child = eval.child(); - if (keptFields.size() > 0) { - // replace the eval with just the kept fields - child = new Eval(eval.source(), eval.child(), keptFields); - } - // put the projection in place - plan = new Project(eval.source(), child, projections); - } - - return plan; - } - } - - /** - * Replace type converting eval with aliasing eval when type change does not occur. - * A following {@link ReplaceAliasingEvalWithProject} will effectively convert {@link ReferenceAttribute} into {@link FieldAttribute}, - * something very useful in local physical planning. - */ - static class ReplaceTrivialTypeConversions extends OptimizerRules.OptimizerRule { - @Override - protected LogicalPlan rule(Eval eval) { - return eval.transformExpressionsOnly(AbstractConvertFunction.class, convert -> { - if (convert.field() instanceof FieldAttribute fa && fa.dataType() == convert.dataType()) { - return fa; - } - return convert; - }); - } - } - - /** - * Rule that removes Aggregate overrides in grouping, aggregates and across them inside. - * The overrides appear when the same alias is used multiple times in aggregations and/or groupings: - * STATS x = COUNT(*), x = MIN(a) BY x = b + 1, x = c + 10 - * becomes - * STATS BY x = c + 10 - * That is the last declaration for a given alias, overrides all the other declarations, with - * groups having priority vs aggregates. - * Separately, it replaces expressions used as group keys inside the aggregates with references: - * STATS max(a + b + 1) BY a + b - * becomes - * STATS max($x + 1) BY $x = a + b - */ - private static class RemoveStatsOverride extends AnalyzerRules.AnalyzerRule { - - @Override - protected boolean skipResolved() { - return false; - } - - @Override - protected LogicalPlan rule(Aggregate agg) { - return agg.resolved() ? removeAggDuplicates(agg) : agg; - } - - private static Aggregate removeAggDuplicates(Aggregate agg) { - var groupings = agg.groupings(); - var aggregates = agg.aggregates(); - - groupings = removeDuplicateNames(groupings); - aggregates = removeDuplicateNames(aggregates); - - // replace EsqlAggregate with Aggregate - return new Aggregate(agg.source(), agg.child(), groupings, aggregates); - } - - private static List removeDuplicateNames(List list) { - var newList = new ArrayList<>(list); - var nameSet = Sets.newHashSetWithExpectedSize(list.size()); - - // remove duplicates - for (int i = list.size() - 1; i >= 0; i--) { - var element = list.get(i); - var name = Expressions.name(element); - if (nameSet.add(name) == false) { - newList.remove(i); - } - } - return newList.size() == list.size() ? list : newList; - } - } - - private abstract static class ParameterizedOptimizerRule extends ParameterizedRule< + public abstract static class ParameterizedOptimizerRule extends ParameterizedRule< SubPlan, LogicalPlan, P> { @@ -1746,114 +302,4 @@ public final LogicalPlan apply(LogicalPlan plan, P context) { protected abstract LogicalPlan rule(SubPlan plan, P context); } - - /** - * Normalize aggregation functions by: - * 1. replaces reference to field attributes with their source - * 2. in case of Count, aligns the various forms (Count(1), Count(0), Count(), Count(*)) to Count(*) - */ - // TODO still needed? - static class NormalizeAggregate extends Rule { - - @Override - public LogicalPlan apply(LogicalPlan plan) { - AttributeMap aliases = new AttributeMap<>(); - - // traverse the tree bottom-up - // 1. if it's Aggregate, normalize the aggregates - // regardless, collect the attributes but only if they refer to an attribute or literal - plan = plan.transformUp(p -> { - if (p instanceof Aggregate agg) { - p = normalize(agg, aliases); - } - p.forEachExpression(Alias.class, a -> { - var child = a.child(); - if (child.foldable() || child instanceof NamedExpression) { - aliases.putIfAbsent(a.toAttribute(), child); - } - }); - - return p; - }); - return plan; - } - - private static LogicalPlan normalize(Aggregate aggregate, AttributeMap aliases) { - var aggs = aggregate.aggregates(); - List newAggs = new ArrayList<>(aggs.size()); - final Holder changed = new Holder<>(false); - - for (NamedExpression agg : aggs) { - var newAgg = (NamedExpression) agg.transformDown(AggregateFunction.class, af -> { - // replace field reference - if (af.field() instanceof NamedExpression ne) { - Attribute attr = ne.toAttribute(); - var resolved = aliases.resolve(attr, attr); - if (resolved != attr) { - changed.set(true); - var newChildren = CollectionUtils.combine(Collections.singletonList(resolved), af.parameters()); - // update the reference so Count can pick it up - af = (AggregateFunction) af.replaceChildren(newChildren); - } - } - // handle Count(*) - if (af instanceof Count count) { - var field = af.field(); - if (field.foldable()) { - var fold = field.fold(); - if (fold != null && StringUtils.WILDCARD.equals(fold) == false) { - changed.set(true); - var source = count.source(); - af = new Count(source, new Literal(source, StringUtils.WILDCARD, DataType.KEYWORD)); - } - } - } - return af; - }); - newAggs.add(newAgg); - } - return changed.get() ? new Aggregate(aggregate.source(), aggregate.child(), aggregate.groupings(), newAggs) : aggregate; - } - } - - public static class FoldNull extends OptimizerRules.FoldNull { - @Override - protected Expression tryReplaceIsNullIsNotNull(Expression e) { - return e; - } - } - - public static class PropagateNullable extends OptimizerRules.PropagateNullable { - protected Expression nullify(Expression exp, Expression nullExp) { - if (exp instanceof Coalesce) { - List newChildren = new ArrayList<>(exp.children()); - newChildren.removeIf(e -> e.semanticEquals(nullExp)); - if (newChildren.size() != exp.children().size() && newChildren.size() > 0) { // coalesce needs at least one input - return exp.replaceChildren(newChildren); - } - } - return Literal.of(exp, null); - } - } - - /** - * Fold the arms of {@code CASE} statements. - * 

    {@code
-     * EVAL c=CASE(true, foo, bar)
-     * }
    - * becomes - * 
    {@code
-     * EVAL c=foo
-     * }
    - */ - static class PartiallyFoldCase extends OptimizerRules.OptimizerExpressionRule { - PartiallyFoldCase() { - super(DOWN); - } - - @Override - protected Expression rule(Case c) { - return c.partiallyFold(); - } - } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/AddDefaultTopN.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/AddDefaultTopN.java new file mode 100644 index 0000000000000..28a7ba4bf7084 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/AddDefaultTopN.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.optimizer.LogicalOptimizerContext; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.TopN; + +/** + * This adds an explicit TopN node to a plan that only has an OrderBy right before Lucene. + * To date, the only known use case that "needs" this is a query of the form + * from test + * | sort emp_no + * | mv_expand first_name + * | rename first_name AS x + * | where x LIKE "*a*" + * | limit 15 + *

    + * or + *

    + * from test + * | sort emp_no + * | mv_expand first_name + * | sort first_name + * | limit 15 + *

    + * PushDownAndCombineLimits rule will copy the "limit 15" after "sort emp_no" if there is no filter on the expanded values + * OR if there is no sort between "limit" and "mv_expand". + * But, since this type of query has such a filter, the "sort emp_no" will have no limit when it reaches the current rule. + */ +public final class AddDefaultTopN extends LogicalPlanOptimizer.ParameterizedOptimizerRule { + + @Override + protected LogicalPlan rule(LogicalPlan plan, LogicalOptimizerContext context) { + if (plan instanceof UnaryPlan unary && unary.child() instanceof OrderBy order && order.child() instanceof EsRelation relation) { + var limit = new Literal(plan.source(), context.configuration().resultTruncationMaxSize(), DataType.INTEGER); + return unary.replaceChild(new TopN(plan.source(), relation, order.order(), limit)); + } + return plan; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplification.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplification.java new file mode 100644 index 0000000000000..b01525cc447fc --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/BooleanSimplification.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Expression; + +public final class BooleanSimplification extends org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.BooleanSimplification { + + public BooleanSimplification() { + super(); + } + + @Override + protected Expression maybeSimplifyNegatable(Expression e) { + return null; + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java index 5cc3184d9ea70..c34252300350c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineDisjunctionsToIn.java @@ -35,7 +35,7 @@ * This rule does NOT check for type compatibility as that phase has been * already be verified in the analyzer. */ -public class CombineDisjunctionsToIn extends org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.OptimizerExpressionRule { +public final class CombineDisjunctionsToIn extends org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.OptimizerExpressionRule { public CombineDisjunctionsToIn() { super(org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.TransformDirection.UP); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineEvals.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineEvals.java new file mode 100644 index 0000000000000..40e9836d0afa1 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineEvals.java @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.util.CollectionUtils; +import org.elasticsearch.xpack.esql.plan.logical.Eval; + +/** + * Combine multiple Evals into one in order to reduce the number of nodes in a plan. + * TODO: eliminate unnecessary fields inside the eval as well + */ +public final class CombineEvals extends OptimizerRules.OptimizerRule { + + public CombineEvals() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + protected LogicalPlan rule(Eval eval) { + LogicalPlan plan = eval; + if (eval.child() instanceof Eval subEval) { + plan = new Eval(eval.source(), subEval.child(), CollectionUtils.combine(subEval.fields(), eval.fields())); + } + return plan; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineProjections.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineProjections.java new file mode 100644 index 0000000000000..940c08ffb97f1 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/CombineProjections.java @@ -0,0 +1,203 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Project; + +import java.util.ArrayList; +import java.util.List; + +public final class CombineProjections extends OptimizerRules.OptimizerRule { + + public CombineProjections() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + @SuppressWarnings("unchecked") + protected LogicalPlan rule(UnaryPlan plan) { + LogicalPlan child = plan.child(); + + if (plan instanceof Project project) { + if (child instanceof Project p) { + // eliminate lower project but first replace the aliases in the upper one + project = p.withProjections(combineProjections(project.projections(), p.projections())); + child = project.child(); + plan = project; + // don't return the plan since the grandchild (now child) might be an aggregate that could not be folded on the way up + // e.g. stats c = count(x) | project c, c as x | project x + // try to apply the rule again opportunistically as another node might be pushed in (a limit might be pushed in) + } + // check if the projection eliminates certain aggregates + // but be mindful of aliases to existing aggregates that we don't want to duplicate to avoid redundant work + if (child instanceof Aggregate a) { + var aggs = a.aggregates(); + var newAggs = projectAggregations(project.projections(), aggs); + // project can be fully removed + if (newAggs != null) { + var newGroups = replacePrunedAliasesUsedInGroupBy(a.groupings(), aggs, newAggs); + plan = new Aggregate(a.source(), a.child(), newGroups, newAggs); + } + } + return plan; + } + + // Agg with underlying Project (group by on sub-queries) + if (plan instanceof Aggregate a) { + if (child instanceof Project p) { + var groupings = a.groupings(); + List groupingAttrs = new ArrayList<>(a.groupings().size()); + for (Expression grouping : groupings) { + if (grouping instanceof Attribute attribute) { + groupingAttrs.add(attribute); + } else { + // After applying ReplaceStatsNestedExpressionWithEval, groupings can only contain attributes. + throw new EsqlIllegalArgumentException("Expected an Attribute, got {}", grouping); + } + } + plan = new Aggregate( + a.source(), + p.child(), + combineUpperGroupingsAndLowerProjections(groupingAttrs, p.projections()), + combineProjections(a.aggregates(), p.projections()) + ); + } + } + + return plan; + } + + // variant of #combineProjections specialized for project followed by agg due to the rewrite rules applied on aggregations + // this method tries to combine the projections by paying attention to: + // - aggregations that are projected away - remove them + // - aliases in the project that point to aggregates - keep them in place (to avoid duplicating the aggs) + private static List projectAggregations( + List upperProjection, + List lowerAggregations + ) { + AttributeSet seen = new AttributeSet(); + for (NamedExpression upper : upperProjection) { + Expression unwrapped = Alias.unwrap(upper); + // projection contains an inner alias (point to an existing fields inside the projection) + if (seen.contains(unwrapped)) { + return null; + } + seen.add(Expressions.attribute(unwrapped)); + } + + lowerAggregations = combineProjections(upperProjection, lowerAggregations); + + return lowerAggregations; + } + + // normally only the upper projections should survive but since the lower list might have aliases definitions + // that might be reused by the upper one, these need to be replaced. + // for example an alias defined in the lower list might be referred in the upper - without replacing it the alias becomes invalid + private static List combineProjections(List upper, List lower) { + + // collect named expressions declaration in the lower list + AttributeMap namedExpressions = new AttributeMap<>(); + // while also collecting the alias map for resolving the source (f1 = 1, f2 = f1, etc..) + AttributeMap aliases = new AttributeMap<>(); + for (NamedExpression ne : lower) { + // record the alias + aliases.put(ne.toAttribute(), Alias.unwrap(ne)); + + // record named expression as is + if (ne instanceof Alias as) { + Expression child = as.child(); + namedExpressions.put(ne.toAttribute(), as.replaceChild(aliases.resolve(child, child))); + } + } + List replaced = new ArrayList<>(); + + // replace any matching attribute with a lower alias (if there's a match) + // but clean-up non-top aliases at the end + for (NamedExpression ne : upper) { + NamedExpression replacedExp = (NamedExpression) ne.transformUp(Attribute.class, a -> namedExpressions.resolve(a, a)); + replaced.add((NamedExpression) trimNonTopLevelAliases(replacedExp)); + } + return replaced; + } + + private static List combineUpperGroupingsAndLowerProjections( + List upperGroupings, + List lowerProjections + ) { + // Collect the alias map for resolving the source (f1 = 1, f2 = f1, etc..) + AttributeMap aliases = new AttributeMap<>(); + for (NamedExpression ne : lowerProjections) { + // Projections are just aliases for attributes, so casting is safe. + aliases.put(ne.toAttribute(), (Attribute) Alias.unwrap(ne)); + } + + // Replace any matching attribute directly with the aliased attribute from the projection. + AttributeSet replaced = new AttributeSet(); + for (Attribute attr : upperGroupings) { + // All substitutions happen before; groupings must be attributes at this point. + replaced.add(aliases.resolve(attr, attr)); + } + return new ArrayList<>(replaced); + } + + /** + * Replace grouping alias previously contained in the aggregations that might have been projected away. + */ + private List replacePrunedAliasesUsedInGroupBy( + List groupings, + List oldAggs, + List newAggs + ) { + AttributeMap removedAliases = new AttributeMap<>(); + AttributeSet currentAliases = new AttributeSet(Expressions.asAttributes(newAggs)); + + // record only removed aliases + for (NamedExpression ne : oldAggs) { + if (ne instanceof Alias alias) { + var attr = ne.toAttribute(); + if (currentAliases.contains(attr) == false) { + removedAliases.put(attr, alias.child()); + } + } + } + + if (removedAliases.isEmpty()) { + return groupings; + } + + var newGroupings = new ArrayList(groupings.size()); + for (Expression group : groupings) { + var transformed = group.transformUp(Attribute.class, a -> removedAliases.resolve(a, a)); + if (Expressions.anyMatch(newGroupings, g -> Expressions.equalsAsAttribute(g, transformed)) == false) { + newGroupings.add(transformed); + } + } + + return newGroupings; + } + + public static Expression trimNonTopLevelAliases(Expression e) { + return e instanceof Alias a ? a.replaceChild(trimAliases(a.child())) : trimAliases(e); + } + + private static Expression trimAliases(Expression e) { + return e.transformDown(Alias.class, Alias::child); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ConvertStringToByteRef.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ConvertStringToByteRef.java new file mode 100644 index 0000000000000..384f56d96de73 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ConvertStringToByteRef.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; + +import java.util.ArrayList; +import java.util.List; + +public final class ConvertStringToByteRef extends OptimizerRules.OptimizerExpressionRule { + + public ConvertStringToByteRef() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + protected Expression rule(Literal lit) { + Object value = lit.value(); + + if (value == null) { + return lit; + } + if (value instanceof String s) { + return Literal.of(lit, new BytesRef(s)); + } + if (value instanceof List l) { + if (l.isEmpty() || false == l.get(0) instanceof String) { + return lit; + } + List byteRefs = new ArrayList<>(l.size()); + for (Object v : l) { + byteRefs.add(new BytesRef(v.toString())); + } + return Literal.of(lit, byteRefs); + } + return lit; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/DuplicateLimitAfterMvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/DuplicateLimitAfterMvExpand.java new file mode 100644 index 0000000000000..6b944bf7adf4f --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/DuplicateLimitAfterMvExpand.java @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.Filter; +import org.elasticsearch.xpack.esql.core.plan.logical.Limit; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.MvExpand; +import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; + +public final class DuplicateLimitAfterMvExpand extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(Limit limit) { + var child = limit.child(); + var shouldSkip = child instanceof Eval + || child instanceof Project + || child instanceof RegexExtract + || child instanceof Enrich + || child instanceof Limit; + + if (shouldSkip == false && child instanceof UnaryPlan unary) { + MvExpand mvExpand = descendantMvExpand(unary); + if (mvExpand != null) { + Limit limitBeforeMvExpand = limitBeforeMvExpand(mvExpand); + // if there is no "appropriate" limit before mv_expand, then push down a copy of the one after it so that: + // - a possible TopN is properly built as low as possible in the tree (closed to Lucene) + // - the input of mv_expand is as small as possible before it is expanded (less rows to inflate and occupy memory) + if (limitBeforeMvExpand == null) { + var duplicateLimit = new Limit(limit.source(), limit.limit(), mvExpand.child()); + return limit.replaceChild(propagateDuplicateLimitUntilMvExpand(duplicateLimit, mvExpand, unary)); + } + } + } + return limit; + } + + private static MvExpand descendantMvExpand(UnaryPlan unary) { + UnaryPlan plan = unary; + AttributeSet filterReferences = new AttributeSet(); + while (plan instanceof Aggregate == false) { + if (plan instanceof MvExpand mve) { + // don't return the mv_expand that has a filter after it which uses the expanded values + // since this will trigger the use of a potentially incorrect (too restrictive) limit further down in the tree + if (filterReferences.isEmpty() == false) { + if (filterReferences.contains(mve.target()) // the same field or reference attribute is used in mv_expand AND filter + || mve.target() instanceof ReferenceAttribute // or the mv_expand attr hasn't yet been resolved to a field attr + // or not all filter references have been resolved to field attributes + || filterReferences.stream().anyMatch(ref -> ref instanceof ReferenceAttribute)) { + return null; + } + } + return mve; + } else if (plan instanceof Filter filter) { + // gather all the filters' references to be checked later when a mv_expand is found + filterReferences.addAll(filter.references()); + } else if (plan instanceof OrderBy) { + // ordering after mv_expand COULD break the order of the results, so the limit shouldn't be copied past mv_expand + // something like from test | sort emp_no | mv_expand job_positions | sort first_name | limit 5 + // (the sort first_name likely changes the order of the docs after sort emp_no, so "limit 5" shouldn't be copied down + return null; + } + + if (plan.child() instanceof UnaryPlan unaryPlan) { + plan = unaryPlan; + } else { + break; + } + } + return null; + } + + private static Limit limitBeforeMvExpand(MvExpand mvExpand) { + UnaryPlan plan = mvExpand; + while (plan instanceof Aggregate == false) { + if (plan instanceof Limit limit) { + return limit; + } + if (plan.child() instanceof UnaryPlan unaryPlan) { + plan = unaryPlan; + } else { + break; + } + } + return null; + } + + private LogicalPlan propagateDuplicateLimitUntilMvExpand(Limit duplicateLimit, MvExpand mvExpand, UnaryPlan child) { + if (child == mvExpand) { + return mvExpand.replaceChild(duplicateLimit); + } else { + return child.replaceChild(propagateDuplicateLimitUntilMvExpand(duplicateLimit, mvExpand, (UnaryPlan) child.child())); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNull.java new file mode 100644 index 0000000000000..25ad5e3966f21 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/FoldNull.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; + +public class FoldNull extends OptimizerRules.FoldNull { + @Override + protected Expression tryReplaceIsNullIsNotNull(Expression e) { + return e; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PartiallyFoldCase.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PartiallyFoldCase.java new file mode 100644 index 0000000000000..6b900d91eb061 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PartiallyFoldCase.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; + +import static org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.TransformDirection.DOWN; + +/** + * Fold the arms of {@code CASE} statements. + * 

    {@code
+ * EVAL c=CASE(true, foo, bar)
+ * }
    + * becomes + * 
    {@code
+ * EVAL c=foo
+ * }
    + */ +public final class PartiallyFoldCase extends OptimizerRules.OptimizerExpressionRule { + public PartiallyFoldCase() { + super(DOWN); + } + + @Override + protected Expression rule(Case c) { + return c.partiallyFold(); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateEmptyRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateEmptyRelation.java new file mode 100644 index 0000000000000..8a3281dd7df81 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateEmptyRelation.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockFactory; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; + +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings("removal") +public class PropagateEmptyRelation extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(UnaryPlan plan) { + LogicalPlan p = plan; + if (plan.child() instanceof LocalRelation local && local.supplier() == LocalSupplier.EMPTY) { + // only care about non-grouped aggs might return something (count) + if (plan instanceof Aggregate agg && agg.groupings().isEmpty()) { + List emptyBlocks = aggsFromEmpty(agg.aggregates()); + p = LogicalPlanOptimizer.skipPlan(plan, LocalSupplier.of(emptyBlocks.toArray(Block[]::new))); + } else { + p = LogicalPlanOptimizer.skipPlan(plan); + } + } + return p; + } + + private List aggsFromEmpty(List aggs) { + List blocks = new ArrayList<>(); + var blockFactory = PlannerUtils.NON_BREAKING_BLOCK_FACTORY; + int i = 0; + for (var agg : aggs) { + // there needs to be an alias + if (Alias.unwrap(agg) instanceof AggregateFunction aggFunc) { + aggOutput(agg, aggFunc, blockFactory, blocks); + } else { + throw new EsqlIllegalArgumentException("Did not expect a non-aliased aggregation {}", agg); + } + } + return blocks; + } + + /** + * The folded aggregation output - this variant is for the coordinator/final. + */ + protected void aggOutput(NamedExpression agg, AggregateFunction aggFunc, BlockFactory blockFactory, List blocks) { + // look for count(literal) with literal != null + Object value = aggFunc instanceof Count count && (count.foldable() == false || count.fold() != null) ? 0L : null; + var wrapper = BlockUtils.wrapperFor(blockFactory, PlannerUtils.toElementType(aggFunc.dataType()), 1); + wrapper.accept(value); + blocks.add(wrapper.builder().build()); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateEvalFoldables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateEvalFoldables.java new file mode 100644 index 0000000000000..872bff80926d6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateEvalFoldables.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.core.plan.logical.Filter; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.rule.Rule; +import org.elasticsearch.xpack.esql.plan.logical.Eval; + +/** + * Replace any reference attribute with its source, if it does not affect the result. + * This avoids ulterior look-ups between attributes and its source across nodes. + */ +public final class PropagateEvalFoldables extends Rule { + + @Override + public LogicalPlan apply(LogicalPlan plan) { + var collectRefs = new AttributeMap(); + + java.util.function.Function replaceReference = r -> collectRefs.resolve(r, r); + + // collect aliases bottom-up + plan.forEachExpressionUp(Alias.class, a -> { + var c = a.child(); + boolean shouldCollect = c.foldable(); + // try to resolve the expression based on an existing foldables + if (shouldCollect == false) { + c = c.transformUp(ReferenceAttribute.class, replaceReference); + shouldCollect = c.foldable(); + } + if (shouldCollect) { + collectRefs.put(a.toAttribute(), Literal.of(c)); + } + }); + if (collectRefs.isEmpty()) { + return plan; + } + + plan = plan.transformUp(p -> { + // Apply the replacement inside Filter and Eval (which shouldn't make a difference) + // TODO: also allow aggregates once aggs on constants are supported. + // C.f. https://github.com/elastic/elasticsearch/issues/100634 + if (p instanceof Filter || p instanceof Eval) { + p = p.transformExpressionsOnly(ReferenceAttribute.class, replaceReference); + } + return p; + }); + + return plan; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullable.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullable.java new file mode 100644 index 0000000000000..73ea21f9c8191 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PropagateNullable.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.expression.function.scalar.nulls.Coalesce; + +import java.util.ArrayList; +import java.util.List; + +public class PropagateNullable extends OptimizerRules.PropagateNullable { + protected Expression nullify(Expression exp, Expression nullExp) { + if (exp instanceof Coalesce) { + List newChildren = new ArrayList<>(exp.children()); + newChildren.removeIf(e -> e.semanticEquals(nullExp)); + if (newChildren.size() != exp.children().size() && newChildren.size() > 0) { // coalesce needs at least one input + return exp.replaceChildren(newChildren); + } + } + return Literal.of(exp, null); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneColumns.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneColumns.java new file mode 100644 index 0000000000000..cb0224c9c834d --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneColumns.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; +import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.plan.logical.Limit; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.rule.Rule; +import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Remove unused columns created in the plan, in fields inside eval or aggregations inside stats. + */ +public final class PruneColumns extends Rule { + + @Override + public LogicalPlan apply(LogicalPlan plan) { + var used = new AttributeSet(); + // don't remove Evals without any Project/Aggregate (which might not occur as the last node in the plan) + var seenProjection = new Holder<>(Boolean.FALSE); + + // start top-to-bottom + // and track used references + var pl = plan.transformDown(p -> { + // skip nodes that simply pass the input through + if (p instanceof Limit) { + return p; + } + + // remember used + boolean recheck; + // analyze the unused items against dedicated 'producer' nodes such as Eval and Aggregate + // perform a loop to retry checking if the current node is completely eliminated + do { + recheck = false; + if (p instanceof Aggregate aggregate) { + var remaining = seenProjection.get() ? removeUnused(aggregate.aggregates(), used) : null; + + if (remaining != null) { + if (remaining.isEmpty()) { + // We still need to have a plan that produces 1 row per group. + if (aggregate.groupings().isEmpty()) { + p = new LocalRelation( + aggregate.source(), + List.of(new EmptyAttribute(aggregate.source())), + LocalSupplier.of( + new Block[] { BlockUtils.constantBlock(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, null, 1) } + ) + ); + } else { + // Aggs cannot produce pages with 0 columns, so retain one grouping. + remaining = List.of(Expressions.attribute(aggregate.groupings().get(0))); + p = new Aggregate(aggregate.source(), aggregate.child(), aggregate.groupings(), remaining); + } + } else { + p = new Aggregate(aggregate.source(), aggregate.child(), aggregate.groupings(), remaining); + } + } + + seenProjection.set(Boolean.TRUE); + } else if (p instanceof Eval eval) { + var remaining = seenProjection.get() ? removeUnused(eval.fields(), used) : null; + // no fields, no eval + if (remaining != null) { + if (remaining.isEmpty()) { + p = eval.child(); + recheck = true; + } else { + p = new Eval(eval.source(), eval.child(), remaining); + } + } + } else if (p instanceof Project) { + seenProjection.set(Boolean.TRUE); + } + } while (recheck); + + used.addAll(p.references()); + + // preserve the state before going to the next node + return p; + }); + + return pl; + } + + /** + * Prunes attributes from the list not found in the given set. + * Returns null if no changed occurred. + */ + private static List removeUnused(List named, AttributeSet used) { + var clone = new ArrayList<>(named); + var it = clone.listIterator(clone.size()); + + // due to Eval, go in reverse + while (it.hasPrevious()) { + N prev = it.previous(); + if (used.contains(prev.toAttribute()) == false) { + it.remove(); + } else { + used.addAll(prev.references()); + } + } + return clone.size() != named.size() ? clone : null; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneEmptyPlans.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneEmptyPlans.java new file mode 100644 index 0000000000000..5c9ef44207366 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneEmptyPlans.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; + +public final class PruneEmptyPlans extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(UnaryPlan plan) { + return plan.output().isEmpty() ? LogicalPlanOptimizer.skipPlan(plan) : plan; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneFilters.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneFilters.java new file mode 100644 index 0000000000000..72df4261663e5 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneFilters.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.Filter; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; + +public final class PruneFilters extends OptimizerRules.PruneFilters { + + @Override + protected LogicalPlan skipPlan(Filter filter) { + return LogicalPlanOptimizer.skipPlan(filter); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneOrderByBeforeStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneOrderByBeforeStats.java new file mode 100644 index 0000000000000..690bc92b1c338 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneOrderByBeforeStats.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.Filter; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; + +public final class PruneOrderByBeforeStats extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(Aggregate agg) { + OrderBy order = findPullableOrderBy(agg.child()); + + LogicalPlan p = agg; + if (order != null) { + p = agg.transformDown(OrderBy.class, o -> o == order ? order.child() : o); + } + return p; + } + + private static OrderBy findPullableOrderBy(LogicalPlan plan) { + OrderBy pullable = null; + if (plan instanceof OrderBy o) { + pullable = o; + } else if (plan instanceof Eval + || plan instanceof Filter + || plan instanceof Project + || plan instanceof RegexExtract + || plan instanceof Enrich) { + pullable = findPullableOrderBy(((UnaryPlan) plan).child()); + } + return pullable; + } + +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneRedundantSortClauses.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneRedundantSortClauses.java new file mode 100644 index 0000000000000..3a9421ee7f159 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PruneRedundantSortClauses.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.ExpressionSet; +import org.elasticsearch.xpack.esql.core.expression.Order; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; + +import java.util.ArrayList; + +public final class PruneRedundantSortClauses extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(OrderBy plan) { + var referencedAttributes = new ExpressionSet(); + var order = new ArrayList(); + for (Order o : plan.order()) { + if (referencedAttributes.add(o)) { + order.add(o); + } + } + + return plan.order().size() == order.size() ? plan : new OrderBy(plan.source(), plan.child(), order); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineFilters.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineFilters.java new file mode 100644 index 0000000000000..647c5c3730157 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineFilters.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.Filter; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; + +public final class PushDownAndCombineFilters extends OptimizerRules.OptimizerRule { + @Override + protected LogicalPlan rule(Filter filter) { + LogicalPlan plan = filter; + LogicalPlan child = filter.child(); + Expression condition = filter.condition(); + + if (child instanceof Filter f) { + // combine nodes into a single Filter with updated ANDed condition + plan = f.with(Predicates.combineAnd(List.of(f.condition(), condition))); + } else if (child instanceof Aggregate agg) { // TODO: re-evaluate along with multi-value support + // Only push [parts of] a filter past an agg if these/it operates on agg's grouping[s], not output. + plan = maybePushDownPastUnary( + filter, + agg, + e -> e instanceof Attribute && agg.output().contains(e) && agg.groupings().contains(e) == false + || e instanceof AggregateFunction + ); + } else if (child instanceof Eval eval) { + // Don't push if Filter (still) contains references of Eval's fields. + var attributes = new AttributeSet(Expressions.asAttributes(eval.fields())); + plan = maybePushDownPastUnary(filter, eval, attributes::contains); + } else if (child instanceof RegexExtract re) { + // Push down filters that do not rely on attributes created by RegexExtract + var attributes = new AttributeSet(Expressions.asAttributes(re.extractedFields())); + plan = maybePushDownPastUnary(filter, re, attributes::contains); + } else if (child instanceof Enrich enrich) { + // Push down filters that do not rely on attributes created by Enrich + var attributes = new AttributeSet(Expressions.asAttributes(enrich.enrichFields())); + plan = maybePushDownPastUnary(filter, enrich, attributes::contains); + } else if (child instanceof Project) { + return LogicalPlanOptimizer.pushDownPastProject(filter); + } else if (child instanceof OrderBy orderBy) { + // swap the filter with its child + plan = orderBy.replaceChild(filter.with(orderBy.child(), condition)); + } + // cannot push past a Limit, this could change the tailing result set returned + return plan; + } + + private static LogicalPlan maybePushDownPastUnary(Filter filter, UnaryPlan unary, Predicate cannotPush) { + LogicalPlan plan; + List pushable = new ArrayList<>(); + List nonPushable = new ArrayList<>(); + for (Expression exp : Predicates.splitAnd(filter.condition())) { + (exp.anyMatch(cannotPush) ? nonPushable : pushable).add(exp); + } + // Push the filter down even if it might not be pushable all the way to ES eventually: eval'ing it closer to the source, + // potentially still in the Exec Engine, distributes the computation. + if (pushable.size() > 0) { + if (nonPushable.size() > 0) { + Filter pushed = new Filter(filter.source(), unary.child(), Predicates.combineAnd(pushable)); + plan = filter.with(unary.replaceChild(pushed), Predicates.combineAnd(nonPushable)); + } else { + plan = unary.replaceChild(filter.with(unary.child(), filter.condition())); + } + } else { + plan = filter; + } + return plan; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineLimits.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineLimits.java new file mode 100644 index 0000000000000..46fb654d03760 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineLimits.java @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.Limit; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.MvExpand; +import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; +import org.elasticsearch.xpack.esql.plan.logical.join.Join; +import org.elasticsearch.xpack.esql.plan.logical.join.JoinType; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; + +public final class PushDownAndCombineLimits extends OptimizerRules.OptimizerRule { + + @Override + public LogicalPlan rule(Limit limit) { + if (limit.child() instanceof Limit childLimit) { + var limitSource = limit.limit(); + var l1 = (int) limitSource.fold(); + var l2 = (int) childLimit.limit().fold(); + return new Limit(limit.source(), Literal.of(limitSource, Math.min(l1, l2)), childLimit.child()); + } else if (limit.child() instanceof UnaryPlan unary) { + if (unary instanceof Eval || unary instanceof Project || unary instanceof RegexExtract || unary instanceof Enrich) { + return unary.replaceChild(limit.replaceChild(unary.child())); + } + // check if there's a 'visible' descendant limit lower than the current one + // and if so, align the current limit since it adds no value + // this applies for cases such as | limit 1 | sort field | limit 10 + else { + Limit descendantLimit = descendantLimit(unary); + if (descendantLimit != null) { + var l1 = (int) limit.limit().fold(); + var l2 = (int) descendantLimit.limit().fold(); + if (l2 <= l1) { + return new Limit(limit.source(), Literal.of(limit.limit(), l2), limit.child()); + } + } + } + } else if (limit.child() instanceof Join join) { + if (join.config().type() == JoinType.LEFT && join.right() instanceof LocalRelation) { + // This is a hash join from something like a lookup. + return join.replaceChildren(limit.replaceChild(join.left()), join.right()); + } + } + return limit; + } + + /** + * Checks the existence of another 'visible' Limit, that exists behind an operation that doesn't produce output more data than + * its input (that is not a relation/source nor aggregation). + * P.S. Typically an aggregation produces less data than the input. + */ + private static Limit descendantLimit(UnaryPlan unary) { + UnaryPlan plan = unary; + while (plan instanceof Aggregate == false) { + if (plan instanceof Limit limit) { + return limit; + } else if (plan instanceof MvExpand) { + // the limit that applies to mv_expand shouldn't be changed + // ie "| limit 1 | mv_expand x | limit 20" where we want that last "limit" to apply on expand results + return null; + } + if (plan.child() instanceof UnaryPlan unaryPlan) { + plan = unaryPlan; + } else { + break; + } + } + return null; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineOrderBy.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineOrderBy.java new file mode 100644 index 0000000000000..f01616953427d --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownAndCombineOrderBy.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.plan.logical.Project; + +public final class PushDownAndCombineOrderBy extends OptimizerRules.OptimizerRule { + @Override + protected LogicalPlan rule(OrderBy orderBy) { + LogicalPlan child = orderBy.child(); + + if (child instanceof OrderBy childOrder) { + // combine orders + return new OrderBy(orderBy.source(), childOrder.child(), orderBy.order()); + } else if (child instanceof Project) { + return LogicalPlanOptimizer.pushDownPastProject(orderBy); + } + + return orderBy; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java new file mode 100644 index 0000000000000..f6a0154108f2d --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEnrich.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.plan.logical.Enrich; + +import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; + +public final class PushDownEnrich extends OptimizerRules.OptimizerRule { + @Override + protected LogicalPlan rule(Enrich en) { + return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(en, asAttributes(en.enrichFields())); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java new file mode 100644 index 0000000000000..b936e5569c950 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownEval.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.plan.logical.Eval; + +import static org.elasticsearch.xpack.esql.core.expression.Expressions.asAttributes; + +public final class PushDownEval extends OptimizerRules.OptimizerRule { + @Override + protected LogicalPlan rule(Eval eval) { + return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(eval, asAttributes(eval.fields())); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java new file mode 100644 index 0000000000000..f247d0a631b29 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/PushDownRegexExtract.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; +import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; + +public final class PushDownRegexExtract extends OptimizerRules.OptimizerRule { + @Override + protected LogicalPlan rule(RegexExtract re) { + return LogicalPlanOptimizer.pushGeneratingPlanPastProjectAndOrderBy(re, re.extractedFields()); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/RemoveStatsOverride.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/RemoveStatsOverride.java new file mode 100644 index 0000000000000..cf04637e456a5 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/RemoveStatsOverride.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.xpack.esql.core.analyzer.AnalyzerRules; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; + +import java.util.ArrayList; +import java.util.List; + +/** + * Rule that removes Aggregate overrides in grouping, aggregates and across them inside. + * The overrides appear when the same alias is used multiple times in aggregations and/or groupings: + * STATS x = COUNT(*), x = MIN(a) BY x = b + 1, x = c + 10 + * becomes + * STATS BY x = c + 10 + * That is the last declaration for a given alias, overrides all the other declarations, with + * groups having priority vs aggregates. + * Separately, it replaces expressions used as group keys inside the aggregates with references: + * STATS max(a + b + 1) BY a + b + * becomes + * STATS max($x + 1) BY $x = a + b + */ +public final class RemoveStatsOverride extends AnalyzerRules.AnalyzerRule { + + @Override + protected boolean skipResolved() { + return false; + } + + @Override + protected LogicalPlan rule(Aggregate agg) { + return agg.resolved() ? removeAggDuplicates(agg) : agg; + } + + private static Aggregate removeAggDuplicates(Aggregate agg) { + var groupings = agg.groupings(); + var aggregates = agg.aggregates(); + + groupings = removeDuplicateNames(groupings); + aggregates = removeDuplicateNames(aggregates); + + // replace EsqlAggregate with Aggregate + return new Aggregate(agg.source(), agg.child(), groupings, aggregates); + } + + private static List removeDuplicateNames(List list) { + var newList = new ArrayList<>(list); + var nameSet = Sets.newHashSetWithExpectedSize(list.size()); + + // remove duplicates + for (int i = list.size() - 1; i >= 0; i--) { + var element = list.get(i); + var name = Expressions.name(element); + if (nameSet.add(name) == false) { + newList.remove(i); + } + } + return newList.size() == list.size() ? list : newList; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceAliasingEvalWithProject.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceAliasingEvalWithProject.java new file mode 100644 index 0000000000000..2bbfeaac965ef --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceAliasingEvalWithProject.java @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.rule.Rule; +import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.Project; + +import java.util.ArrayList; +import java.util.List; + +/** + * Replace aliasing evals (eval x=a) with a projection which can be further combined / simplified. + * The rule gets applied only if there's another project (Project/Stats) above it. + *

    + * Needs to take into account shadowing of potentially intermediate fields: + * eval x = a + 1, y = x, z = y + 1, y = z, w = y + 1 + * The output should be + * eval x = a + 1, z = a + 1 + 1, w = a + 1 + 1 + * project x, z, z as y, w + */ +public final class ReplaceAliasingEvalWithProject extends Rule { + + @Override + public LogicalPlan apply(LogicalPlan logicalPlan) { + Holder enabled = new Holder<>(false); + + return logicalPlan.transformDown(p -> { + // found projection, turn enable flag on + if (p instanceof Aggregate || p instanceof Project) { + enabled.set(true); + } else if (enabled.get() && p instanceof Eval eval) { + p = rule(eval); + } + + return p; + }); + } + + private LogicalPlan rule(Eval eval) { + LogicalPlan plan = eval; + + // holds simple aliases such as b = a, c = b, d = c + AttributeMap basicAliases = new AttributeMap<>(); + // same as above but keeps the original expression + AttributeMap basicAliasSources = new AttributeMap<>(); + + List keptFields = new ArrayList<>(); + + var fields = eval.fields(); + for (int i = 0, size = fields.size(); i < size; i++) { + Alias field = fields.get(i); + Expression child = field.child(); + var attribute = field.toAttribute(); + // put the aliases in a separate map to separate the underlying resolve from other aliases + if (child instanceof Attribute) { + basicAliases.put(attribute, child); + basicAliasSources.put(attribute, field); + } else { + // be lazy and start replacing name aliases only if needed + if (basicAliases.size() > 0) { + // update the child through the field + field = (Alias) field.transformUp(e -> basicAliases.resolve(e, e)); + } + keptFields.add(field); + } + } + + // at least one alias encountered, move it into a project + if (basicAliases.size() > 0) { + // preserve the eval output (takes care of shadowing and order) but replace the basic aliases + List projections = new ArrayList<>(eval.output()); + // replace the removed aliases with their initial definition - however use the output to preserve the shadowing + for (int i = projections.size() - 1; i >= 0; i--) { + NamedExpression project = projections.get(i); + projections.set(i, basicAliasSources.getOrDefault(project, project)); + } + + LogicalPlan child = eval.child(); + if (keptFields.size() > 0) { + // replace the eval with just the kept fields + child = new Eval(eval.source(), eval.child(), keptFields); + } + // put the projection in place + plan = new Project(eval.source(), child, projections); + } + + return plan; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceLimitAndSortAsTopN.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceLimitAndSortAsTopN.java new file mode 100644 index 0000000000000..ec912735f8451 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceLimitAndSortAsTopN.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.Limit; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.plan.logical.TopN; + +public final class ReplaceLimitAndSortAsTopN extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(Limit plan) { + LogicalPlan p = plan; + if (plan.child() instanceof OrderBy o) { + p = new TopN(plan.source(), o.child(), o.order(), plan.limit()); + } + return p; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceLookupWithJoin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceLookupWithJoin.java new file mode 100644 index 0000000000000..f6c8f4a59a70c --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceLookupWithJoin.java @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.Lookup; +import org.elasticsearch.xpack.esql.plan.logical.join.Join; + +public final class ReplaceLookupWithJoin extends OptimizerRules.OptimizerRule { + + public ReplaceLookupWithJoin() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + protected LogicalPlan rule(Lookup lookup) { + // left join between the main relation and the local, lookup relation + return new Join(lookup.source(), lookup.child(), lookup.localRelation(), lookup.joinConfig()); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java new file mode 100644 index 0000000000000..476da7476f7fb --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceOrderByExpressionWithEval.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Order; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.Project; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.xpack.esql.optimizer.rules.SubstituteSurrogates.rawTemporaryName; + +public final class ReplaceOrderByExpressionWithEval extends OptimizerRules.OptimizerRule { + private static int counter = 0; + + @Override + protected LogicalPlan rule(OrderBy orderBy) { + int size = orderBy.order().size(); + List evals = new ArrayList<>(size); + List newOrders = new ArrayList<>(size); + + for (int i = 0; i < size; i++) { + var order = orderBy.order().get(i); + if (order.child() instanceof Attribute == false) { + var name = rawTemporaryName("order_by", String.valueOf(i), String.valueOf(counter++)); + var eval = new Alias(order.child().source(), name, order.child()); + newOrders.add(order.replaceChildren(List.of(eval.toAttribute()))); + evals.add(eval); + } else { + newOrders.add(order); + } + } + if (evals.isEmpty()) { + return orderBy; + } else { + var newOrderBy = new OrderBy(orderBy.source(), new Eval(orderBy.source(), orderBy.child(), evals), newOrders); + return new Project(orderBy.source(), newOrderBy, orderBy.output()); + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceRegexMatch.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceRegexMatch.java new file mode 100644 index 0000000000000..5cba7349debfd --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceRegexMatch.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RegexMatch; +import org.elasticsearch.xpack.esql.core.expression.predicate.regex.StringPattern; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; + +public final class ReplaceRegexMatch extends org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.OptimizerExpressionRule< + RegexMatch> { + + public ReplaceRegexMatch() { + super(org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules.TransformDirection.DOWN); + } + + @Override + public Expression rule(RegexMatch regexMatch) { + Expression e = regexMatch; + StringPattern pattern = regexMatch.pattern(); + if (pattern.matchesAll()) { + e = new IsNotNull(e.source(), regexMatch.field()); + } else { + String match = pattern.exactMatch(); + if (match != null) { + Literal literal = new Literal(regexMatch.source(), match, DataType.KEYWORD); + e = regexToEquals(regexMatch, literal); + } + } + return e; + } + + protected Expression regexToEquals(RegexMatch regexMatch, Literal literal) { + return new Equals(regexMatch.source(), regexMatch.field(), literal); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java new file mode 100644 index 0000000000000..9a24926953947 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsAggExpressionWithEval.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.AttributeMap; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.CollectionUtils; +import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.Project; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singleton; + +/** + * Replace nested expressions over aggregates with synthetic eval post the aggregation + * stats a = sum(a) + min(b) by x + * becomes + * stats a1 = sum(a), a2 = min(b) by x | eval a = a1 + a2 | keep a, x + * The rule also considers expressions applied over groups: + * stats a = x + 1 by x becomes stats by x | eval a = x + 1 | keep a, x + * And to combine the two: + * stats a = x + count(*) by x + * becomes + * stats a1 = count(*) by x | eval a = x + a1 | keep a1, x + * Since the logic is very similar, this rule also handles duplicate aggregate functions to avoid duplicate compute + * stats a = min(x), b = min(x), c = count(*), d = count() by g + * becomes + * stats a = min(x), c = count(*) by g | eval b = a, d = c | keep a, b, c, d, g + */ +public final class ReplaceStatsAggExpressionWithEval extends OptimizerRules.OptimizerRule { + public ReplaceStatsAggExpressionWithEval() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + protected LogicalPlan rule(Aggregate aggregate) { + // build alias map + AttributeMap aliases = new AttributeMap<>(); + aggregate.forEachExpressionUp(Alias.class, a -> aliases.put(a.toAttribute(), a.child())); + + // break down each aggregate into AggregateFunction and/or grouping key + // preserve the projection at the end + List aggs = aggregate.aggregates(); + + // root/naked aggs + Map rootAggs = Maps.newLinkedHashMapWithExpectedSize(aggs.size()); + // evals (original expression relying on multiple aggs) + List newEvals = new ArrayList<>(); + List newProjections = new ArrayList<>(); + // track the aggregate aggs (including grouping which is not an AggregateFunction) + List newAggs = new ArrayList<>(); + + Holder changed = new Holder<>(false); + int[] counter = new int[] { 0 }; + + for (NamedExpression agg : aggs) { + if (agg instanceof Alias as) { + // if the child a nested expression + Expression child = as.child(); + + // common case - handle duplicates + if (child instanceof AggregateFunction af) { + AggregateFunction canonical = (AggregateFunction) af.canonical(); + Expression field = canonical.field().transformUp(e -> aliases.resolve(e, e)); + canonical = (AggregateFunction) canonical.replaceChildren( + CollectionUtils.combine(singleton(field), canonical.parameters()) + ); + + Alias found = rootAggs.get(canonical); + // aggregate is new + if (found == null) { + rootAggs.put(canonical, as); + newAggs.add(as); + newProjections.add(as.toAttribute()); + } + // agg already exists - preserve the current alias but point it to the existing agg + // thus don't add it to the list of aggs as we don't want duplicated compute + else { + changed.set(true); + newProjections.add(as.replaceChild(found.toAttribute())); + } + } + // nested expression over aggregate function or groups + // replace them with reference and move the expression into a follow-up eval + else { + changed.set(true); + Expression aggExpression = child.transformUp(AggregateFunction.class, af -> { + AggregateFunction canonical = (AggregateFunction) af.canonical(); + Alias alias = rootAggs.get(canonical); + if (alias == null) { + // create synthetic alias ove the found agg function + alias = new Alias( + af.source(), + syntheticName(canonical, child, counter[0]++), + as.qualifier(), + canonical, + null, + true + ); + // and remember it to remove duplicates + rootAggs.put(canonical, alias); + // add it to the list of aggregates and continue + newAggs.add(alias); + } + // (even when found) return a reference to it + return alias.toAttribute(); + }); + + Alias alias = as.replaceChild(aggExpression); + newEvals.add(alias); + newProjections.add(alias.toAttribute()); + } + } + // not an alias (e.g. grouping field) + else { + newAggs.add(agg); + newProjections.add(agg.toAttribute()); + } + } + + LogicalPlan plan = aggregate; + if (changed.get()) { + Source source = aggregate.source(); + plan = new Aggregate(source, aggregate.child(), aggregate.groupings(), newAggs); + if (newEvals.size() > 0) { + plan = new Eval(source, plan, newEvals); + } + // preserve initial projection + plan = new Project(source, plan, newProjections); + } + + return plan; + } + + static String syntheticName(Expression expression, Expression af, int counter) { + return SubstituteSurrogates.temporaryName(expression, af, counter); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java new file mode 100644 index 0000000000000..dc7686f57f2f4 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceStatsNestedExpressionWithEval.java @@ -0,0 +1,137 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Eval; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Replace nested expressions inside an aggregate with synthetic eval (which end up being projected away by the aggregate). + * stats sum(a + 1) by x % 2 + * becomes + * eval `a + 1` = a + 1, `x % 2` = x % 2 | stats sum(`a+1`_ref) by `x % 2`_ref + */ +public final class ReplaceStatsNestedExpressionWithEval extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(Aggregate aggregate) { + List evals = new ArrayList<>(); + Map evalNames = new HashMap<>(); + Map groupingAttributes = new HashMap<>(); + List newGroupings = new ArrayList<>(aggregate.groupings()); + boolean groupingChanged = false; + + // start with the groupings since the aggs might duplicate it + for (int i = 0, s = newGroupings.size(); i < s; i++) { + Expression g = newGroupings.get(i); + // move the alias into an eval and replace it with its attribute + if (g instanceof Alias as) { + groupingChanged = true; + var attr = as.toAttribute(); + evals.add(as); + evalNames.put(as.name(), attr); + newGroupings.set(i, attr); + if (as.child() instanceof GroupingFunction gf) { + groupingAttributes.put(gf, attr); + } + } + } + + Holder aggsChanged = new Holder<>(false); + List aggs = aggregate.aggregates(); + List newAggs = new ArrayList<>(aggs.size()); + + // map to track common expressions + Map expToAttribute = new HashMap<>(); + for (Alias a : evals) { + expToAttribute.put(a.child().canonical(), a.toAttribute()); + } + + int[] counter = new int[] { 0 }; + // for the aggs make sure to unwrap the agg function and check the existing groupings + for (NamedExpression agg : aggs) { + NamedExpression a = (NamedExpression) agg.transformDown(Alias.class, as -> { + // if the child is a nested expression + Expression child = as.child(); + + // shortcut for common scenario + if (child instanceof AggregateFunction af && af.field() instanceof Attribute) { + return as; + } + + // check if the alias matches any from grouping otherwise unwrap it + Attribute ref = evalNames.get(as.name()); + if (ref != null) { + aggsChanged.set(true); + return ref; + } + + // 1. look for the aggregate function + var replaced = child.transformUp(AggregateFunction.class, af -> { + Expression result = af; + + Expression field = af.field(); + // 2. if the field is a nested expression (not attribute or literal), replace it + if (field instanceof Attribute == false && field.foldable() == false) { + // 3. create a new alias if one doesn't exist yet no reference + Attribute attr = expToAttribute.computeIfAbsent(field.canonical(), k -> { + Alias newAlias = new Alias(k.source(), syntheticName(k, af, counter[0]++), null, k, null, true); + evals.add(newAlias); + return newAlias.toAttribute(); + }); + aggsChanged.set(true); + // replace field with attribute + List newChildren = new ArrayList<>(af.children()); + newChildren.set(0, attr); + result = af.replaceChildren(newChildren); + } + return result; + }); + // replace any grouping functions with their references pointing to the added synthetic eval + replaced = replaced.transformDown(GroupingFunction.class, gf -> { + aggsChanged.set(true); + // should never return null, as it's verified. + // but even if broken, the transform will fail safely; otoh, returning `gf` will fail later due to incorrect plan. + return groupingAttributes.get(gf); + }); + + return as.replaceChild(replaced); + }); + + newAggs.add(a); + } + + if (evals.size() > 0) { + var groupings = groupingChanged ? newGroupings : aggregate.groupings(); + var aggregates = aggsChanged.get() ? newAggs : aggregate.aggregates(); + + var newEval = new Eval(aggregate.source(), aggregate.child(), evals); + aggregate = new Aggregate(aggregate.source(), newEval, groupings, aggregates); + } + + return aggregate; + } + + static String syntheticName(Expression expression, AggregateFunction af, int counter) { + return SubstituteSurrogates.temporaryName(expression, af, counter); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceTrivialTypeConversions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceTrivialTypeConversions.java new file mode 100644 index 0000000000000..2763c71c4bcb6 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/ReplaceTrivialTypeConversions.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; +import org.elasticsearch.xpack.esql.plan.logical.Eval; + +/** + * Replace type converting eval with aliasing eval when type change does not occur. + * A following {@link ReplaceAliasingEvalWithProject} will effectively convert {@link ReferenceAttribute} into {@link FieldAttribute}, + * something very useful in local physical planning. + */ +public final class ReplaceTrivialTypeConversions extends OptimizerRules.OptimizerRule { + @Override + protected LogicalPlan rule(Eval eval) { + return eval.transformExpressionsOnly(AbstractConvertFunction.class, convert -> { + if (convert.field() instanceof FieldAttribute fa && fa.dataType() == convert.dataType()) { + return fa; + } + return convert; + }); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SkipQueryOnEmptyMappings.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SkipQueryOnEmptyMappings.java new file mode 100644 index 0000000000000..7ec215db65626 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SkipQueryOnEmptyMappings.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; + +public final class SkipQueryOnEmptyMappings extends OptimizerRules.OptimizerRule { + + @Override + protected LogicalPlan rule(EsRelation plan) { + return plan.index().concreteIndices().isEmpty() ? new LocalRelation(plan.source(), plan.output(), LocalSupplier.EMPTY) : plan; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SkipQueryOnLimitZero.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SkipQueryOnLimitZero.java new file mode 100644 index 0000000000000..7cb4f2926045d --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SkipQueryOnLimitZero.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.Limit; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer; + +public final class SkipQueryOnLimitZero extends OptimizerRules.SkipQueryOnLimitZero { + + @Override + protected LogicalPlan skipPlan(Limit limit) { + return LogicalPlanOptimizer.skipPlan(limit); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SplitInWithFoldableValue.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SplitInWithFoldableValue.java new file mode 100644 index 0000000000000..c762f396a6f43 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SplitInWithFoldableValue.java @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Or; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In; + +import java.util.ArrayList; +import java.util.List; + +/** + * 3 in (field, 4, 5) --> 3 in (field) or 3 in (4, 5) + */ +public final class SplitInWithFoldableValue extends OptimizerRules.OptimizerExpressionRule { + + public SplitInWithFoldableValue() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + public Expression rule(In in) { + if (in.value().foldable()) { + List foldables = new ArrayList<>(in.list().size()); + List nonFoldables = new ArrayList<>(in.list().size()); + in.list().forEach(e -> { + if (e.foldable() && Expressions.isNull(e) == false) { // keep `null`s, needed for the 3VL + foldables.add(e); + } else { + nonFoldables.add(e); + } + }); + if (foldables.size() > 0 && nonFoldables.size() > 0) { + In withFoldables = new In(in.source(), in.value(), foldables); + In withoutFoldables = new In(in.source(), in.value(), nonFoldables); + return new Or(in.source(), withFoldables, withoutFoldables); + } + } + return in; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSpatialSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSpatialSurrogates.java new file mode 100644 index 0000000000000..c5293785bf1ba --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSpatialSurrogates.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.SpatialRelatesFunction; + +/** + * Currently this works similarly to SurrogateExpression, leaving the logic inside the expressions, + * so each can decide for itself whether or not to change to a surrogate expression. + * But what is actually being done is similar to LiteralsOnTheRight. We can consider in the future moving + * this in either direction, reducing the number of rules, but for now, + * it's a separate rule to reduce the risk of unintended interactions with other rules. + */ +public final class SubstituteSpatialSurrogates extends OptimizerRules.OptimizerExpressionRule { + + public SubstituteSpatialSurrogates() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + protected SpatialRelatesFunction rule(SpatialRelatesFunction function) { + return function.surrogate(); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java new file mode 100644 index 0000000000000..39617b443a286 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/SubstituteSurrogates.java @@ -0,0 +1,155 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.optimizer.rules; + +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BlockUtils; +import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; +import org.elasticsearch.xpack.esql.core.optimizer.OptimizerRules; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.expression.SurrogateExpression; +import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; +import org.elasticsearch.xpack.esql.plan.logical.Eval; +import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation; +import org.elasticsearch.xpack.esql.plan.logical.local.LocalSupplier; +import org.elasticsearch.xpack.esql.planner.PlannerUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public final class SubstituteSurrogates extends OptimizerRules.OptimizerRule { + // TODO: currently this rule only works for aggregate functions (AVG) + + public SubstituteSurrogates() { + super(OptimizerRules.TransformDirection.UP); + } + + @Override + protected LogicalPlan rule(Aggregate aggregate) { + var aggs = aggregate.aggregates(); + List newAggs = new ArrayList<>(aggs.size()); + // existing aggregate and their respective attributes + Map aggFuncToAttr = new HashMap<>(); + // surrogate functions eval + List transientEval = new ArrayList<>(); + boolean changed = false; + + // first pass to check existing aggregates (to avoid duplication and alias waste) + for (NamedExpression agg : aggs) { + if (Alias.unwrap(agg) instanceof AggregateFunction af) { + if ((af instanceof SurrogateExpression se && se.surrogate() != null) == false) { + aggFuncToAttr.put(af, agg.toAttribute()); + } + } + } + + int[] counter = new int[] { 0 }; + // 0. check list of surrogate expressions + for (NamedExpression agg : aggs) { + Expression e = Alias.unwrap(agg); + if (e instanceof SurrogateExpression sf && sf.surrogate() != null) { + changed = true; + Expression s = sf.surrogate(); + + // if the expression is NOT a 1:1 replacement need to add an eval + if (s instanceof AggregateFunction == false) { + // 1. collect all aggregate functions from the expression + var surrogateWithRefs = s.transformUp(AggregateFunction.class, af -> { + // 2. check if they are already use otherwise add them to the Aggregate with some made-up aliases + // 3. replace them inside the expression using the given alias + var attr = aggFuncToAttr.get(af); + // the agg doesn't exist in the Aggregate, create an alias for it and save its attribute + if (attr == null) { + var temporaryName = temporaryName(af, agg, counter[0]++); + // create a synthetic alias (so it doesn't clash with a user defined name) + var newAlias = new Alias(agg.source(), temporaryName, null, af, null, true); + attr = newAlias.toAttribute(); + aggFuncToAttr.put(af, attr); + newAggs.add(newAlias); + } + return attr; + }); + // 4. move the expression as an eval using the original alias + // copy the original alias id so that other nodes using it down stream (e.g. eval referring to the original agg) + // don't have to updated + var aliased = new Alias(agg.source(), agg.name(), null, surrogateWithRefs, agg.toAttribute().id()); + transientEval.add(aliased); + } + // the replacement is another aggregate function, so replace it in place + else { + newAggs.add((NamedExpression) agg.replaceChildren(Collections.singletonList(s))); + } + } else { + newAggs.add(agg); + } + } + + LogicalPlan plan = aggregate; + if (changed) { + var source = aggregate.source(); + if (newAggs.isEmpty() == false) { + plan = new Aggregate(source, aggregate.child(), aggregate.groupings(), newAggs); + } else { + // All aggs actually have been surrogates for (foldable) expressions, e.g. + // \_Aggregate[[],[AVG([1, 2][INTEGER]) AS s]] + // Replace by a local relation with one row, followed by an eval, e.g. + // \_Eval[[MVAVG([1, 2][INTEGER]) AS s]] + // \_LocalRelation[[{e}#21],[ConstantNullBlock[positions=1]]] + plan = new LocalRelation( + source, + List.of(new EmptyAttribute(source)), + LocalSupplier.of(new Block[] { BlockUtils.constantBlock(PlannerUtils.NON_BREAKING_BLOCK_FACTORY, null, 1) }) + ); + } + // 5. force the initial projection in place + if (transientEval.isEmpty() == false) { + plan = new Eval(source, plan, transientEval); + // project away transient fields and re-enforce the original order using references (not copies) to the original aggs + // this works since the replaced aliases have their nameId copied to avoid having to update all references (which has + // a cascading effect) + plan = new Project(source, plan, Expressions.asAttributes(aggs)); + } + } + + return plan; + } + + public static String temporaryName(Expression inner, Expression outer, int suffix) { + String in = toString(inner); + String out = toString(outer); + return rawTemporaryName(in, out, String.valueOf(suffix)); + } + + public static String rawTemporaryName(String inner, String outer, String suffix) { + return "$$" + inner + "$" + outer + "$" + suffix; + } + + static int TO_STRING_LIMIT = 16; + + static String toString(Expression ex) { + return ex instanceof AggregateFunction af ? af.functionName() : extractString(ex); + } + + static String extractString(Expression ex) { + return ex instanceof NamedExpression ne ? ne.name() : limitToString(ex.sourceText()).replace(' ', '_'); + } + + static String limitToString(String string) { + return string.length() > 16 ? string.substring(0, TO_STRING_LIMIT - 1) + ">" : string; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java index 863476ba55686..0d45ce10b1966 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/package-info.java @@ -29,6 +29,7 @@ * functions, designed to run over a {@link org.elasticsearch.compute.data.Block}

    • *
  • {@link org.elasticsearch.xpack.esql.session.EsqlSession} - manages state across a query
    • *
  • {@link org.elasticsearch.xpack.esql.expression.function.scalar} - Guide to writing scalar functions
    • + *
  • {@link org.elasticsearch.xpack.esql.expression.function.aggregate} - Guide to writing aggregation functions
    • *
  • {@link org.elasticsearch.xpack.esql.analysis.Analyzer} - The first step in query processing
    • *
  • {@link org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer} - Coordinator level logical optimizations
    • *
  • {@link org.elasticsearch.xpack.esql.optimizer.LocalLogicalPlanOptimizer} - Data node level logical optimizations
    • diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java index 59801e59555b5..41db2aa54387b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/ExpressionBuilder.java @@ -58,7 +58,6 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.math.BigInteger; import java.time.Duration; @@ -549,7 +548,7 @@ public Expression visitInlineCast(EsqlBaseParser.InlineCastContext ctx) { @Override public DataType visitToDataType(EsqlBaseParser.ToDataTypeContext ctx) { String typeName = visitIdentifier(ctx.identifier()); - DataType dataType = EsqlDataTypes.fromNameOrAlias(typeName); + DataType dataType = DataType.fromNameOrAlias(typeName); if (dataType == DataType.UNSUPPORTED) { throw new ParsingException(source(ctx), "Unknown data type named [{}]", typeName); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java index b7e4fc9ae622f..08916c14e91bf 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/EsRelation.java @@ -92,6 +92,8 @@ public List output() { @Override public boolean expressionsResolved() { + // For unresolved expressions to exist in EsRelation is fine, as long as they are not used in later operations + // This allows for them to be converted to null@unsupported fields in final output, an important feature of ES|QL return true; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java index 97ba6feb6e278..8b9b5398b3cec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/EstimatesRowSize.java @@ -113,6 +113,7 @@ static int estimateSize(DataType dataType) { default -> 50; // wild estimate for the size of a string. }; case DOC -> throw new EsqlIllegalArgumentException("can't load a [doc] with field extraction"); + case FLOAT -> Float.BYTES; case DOUBLE -> Double.BYTES; case INT -> Integer.BYTES; case LONG -> Long.BYTES; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java index 7c124701fe332..dff0a6f0eade3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/HashJoinExec.java @@ -13,7 +13,7 @@ import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; -import org.elasticsearch.xpack.esql.io.stream.PlanNamedTypes; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; @@ -53,7 +53,7 @@ public HashJoinExec(PlanStreamInput in) throws IOException { super(Source.readFrom(in), in.readPhysicalPlanNode()); this.joinData = new LocalSourceExec(in); this.matchFields = in.readNamedWriteableCollectionAsList(NamedExpression.class); - this.conditions = in.readCollectionAsList(i -> (Equals) PlanNamedTypes.readBinComparison(in, "equals")); + this.conditions = in.readCollectionAsList(i -> (Equals) EsqlBinaryComparison.readFrom(in)); this.output = in.readNamedWriteableCollectionAsList(Attribute.class); } @@ -62,7 +62,7 @@ public void writeTo(PlanStreamOutput out) throws IOException { out.writePhysicalPlanNode(child()); joinData.writeTo(out); out.writeNamedWriteableCollection(matchFields); - out.writeCollection(conditions, (o, v) -> PlanNamedTypes.writeBinComparison(out, v)); + out.writeCollection(conditions, (o, v) -> v.writeTo(o)); out.writeNamedWriteableCollection(output); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java index 68e6ea4d6cadb..83fdd5dc0c5d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AggregateMapper.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialAggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.SpatialCentroid; import org.elasticsearch.xpack.esql.expression.function.aggregate.Sum; +import org.elasticsearch.xpack.esql.expression.function.aggregate.TopList; import org.elasticsearch.xpack.esql.expression.function.aggregate.Values; import java.lang.invoke.MethodHandle; @@ -61,7 +62,8 @@ final class AggregateMapper { Percentile.class, SpatialCentroid.class, Sum.class, - Values.class + Values.class, + TopList.class ); /** Record of agg Class, type, and grouping (or non-grouping). */ @@ -143,6 +145,8 @@ private static Stream, Tuple>> typeAndNames(Class } else if (Values.class.isAssignableFrom(clazz)) { // TODO can't we figure this out from the function itself? types = List.of("Int", "Long", "Double", "Boolean", "BytesRef"); + } else if (TopList.class.isAssignableFrom(clazz)) { + types = List.of("Int", "Long", "Double"); } else { assert clazz == CountDistinct.class : "Expected CountDistinct, got: " + clazz; types = Stream.concat(NUMERIC.stream(), Stream.of("Boolean", "BytesRef")).toList(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 04ed433200c2f..fdba785f668d7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -7,20 +7,28 @@ package org.elasticsearch.xpack.esql.planner; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.elasticsearch.common.breaker.CircuitBreaker; +import org.elasticsearch.common.breaker.NoopCircuitBreaker; import org.elasticsearch.common.logging.HeaderWarning; +import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.compute.aggregation.GroupingAggregator; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.ElementType; +import org.elasticsearch.compute.data.Page; import org.elasticsearch.compute.lucene.LuceneCountOperator; import org.elasticsearch.compute.lucene.LuceneOperator; import org.elasticsearch.compute.lucene.LuceneSourceOperator; import org.elasticsearch.compute.lucene.LuceneTopNSourceOperator; import org.elasticsearch.compute.lucene.TimeSeriesSortedSourceOperatorFactory; import org.elasticsearch.compute.lucene.ValuesSourceReaderOperator; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; import org.elasticsearch.compute.operator.Operator; import org.elasticsearch.compute.operator.OrdinalsGroupingOperator; import org.elasticsearch.compute.operator.SourceOperator; @@ -35,13 +43,16 @@ import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.search.NestedHelper; +import org.elasticsearch.search.fetch.StoredFieldsSpec; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.plan.physical.AggregateExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec.FieldSort; @@ -50,6 +61,7 @@ import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.LocalExecutionPlannerContext; import org.elasticsearch.xpack.esql.planner.LocalExecutionPlanner.PhysicalOperation; import org.elasticsearch.xpack.esql.type.EsqlDataTypes; +import org.elasticsearch.xpack.esql.type.MultiTypeEsField; import java.io.IOException; import java.util.ArrayList; @@ -102,17 +114,42 @@ public final PhysicalOperation fieldExtractPhysicalOperation(FieldExtractExec fi var docValuesAttrs = fieldExtractExec.docValuesAttributes(); for (Attribute attr : fieldExtractExec.attributesToExtract()) { layout.append(attr); + var unionTypes = findUnionTypes(attr); DataType dataType = attr.dataType(); MappedFieldType.FieldExtractPreference fieldExtractPreference = PlannerUtils.extractPreference(docValuesAttrs.contains(attr)); ElementType elementType = PlannerUtils.toElementType(dataType, fieldExtractPreference); String fieldName = attr.name(); boolean isUnsupported = EsqlDataTypes.isUnsupported(dataType); - IntFunction loader = s -> shardContexts.get(s).blockLoader(fieldName, isUnsupported, fieldExtractPreference); + IntFunction loader = s -> getBlockLoaderFor(s, fieldName, isUnsupported, fieldExtractPreference, unionTypes); fields.add(new ValuesSourceReaderOperator.FieldInfo(fieldName, elementType, loader)); } return source.with(new ValuesSourceReaderOperator.Factory(fields, readers, docChannel), layout.build()); } + private BlockLoader getBlockLoaderFor( + int shardId, + String fieldName, + boolean isUnsupported, + MappedFieldType.FieldExtractPreference fieldExtractPreference, + MultiTypeEsField unionTypes + ) { + DefaultShardContext shardContext = (DefaultShardContext) shardContexts.get(shardId); + BlockLoader blockLoader = shardContext.blockLoader(fieldName, isUnsupported, fieldExtractPreference); + if (unionTypes != null) { + String indexName = shardContext.ctx.index().getName(); + Expression conversion = unionTypes.getConversionExpressionForIndex(indexName); + return new TypeConvertingBlockLoader(blockLoader, (AbstractConvertFunction) conversion); + } + return blockLoader; + } + + private MultiTypeEsField findUnionTypes(Attribute attr) { + if (attr instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField multiTypeEsField) { + return multiTypeEsField; + } + return null; + } + public Function querySupplier(QueryBuilder builder) { QueryBuilder qb = builder == null ? QueryBuilders.matchAllQuery() : builder; return ctx -> shardContexts.get(ctx.index()).toQuery(qb); @@ -321,4 +358,96 @@ public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() { return loader; } } + + static class TypeConvertingBlockLoader implements BlockLoader { + protected final BlockLoader delegate; + private final EvalOperator.ExpressionEvaluator convertEvaluator; + + protected TypeConvertingBlockLoader(BlockLoader delegate, AbstractConvertFunction convertFunction) { + this.delegate = delegate; + DriverContext driverContext1 = new DriverContext( + BigArrays.NON_RECYCLING_INSTANCE, + new org.elasticsearch.compute.data.BlockFactory( + new NoopCircuitBreaker(CircuitBreaker.REQUEST), + BigArrays.NON_RECYCLING_INSTANCE + ) + ); + this.convertEvaluator = convertFunction.toEvaluator(e -> driverContext -> new EvalOperator.ExpressionEvaluator() { + @Override + public org.elasticsearch.compute.data.Block eval(Page page) { + // This is a pass-through evaluator, since it sits directly on the source loading (no prior expressions) + return page.getBlock(0); + } + + @Override + public void close() {} + }).get(driverContext1); + } + + @Override + public Builder builder(BlockFactory factory, int expectedCount) { + // Return the delegates builder, which can build the original mapped type, before conversion + return delegate.builder(factory, expectedCount); + } + + @Override + public Block convert(Block block) { + Page page = new Page((org.elasticsearch.compute.data.Block) block); + return convertEvaluator.eval(page); + } + + @Override + public ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) throws IOException { + ColumnAtATimeReader reader = delegate.columnAtATimeReader(context); + if (reader == null) { + return null; + } + return new ColumnAtATimeReader() { + @Override + public Block read(BlockFactory factory, Docs docs) throws IOException { + Block block = reader.read(factory, docs); + Page page = new Page((org.elasticsearch.compute.data.Block) block); + org.elasticsearch.compute.data.Block converted = convertEvaluator.eval(page); + return converted; + } + + @Override + public boolean canReuse(int startingDocID) { + return reader.canReuse(startingDocID); + } + + @Override + public String toString() { + return reader.toString(); + } + }; + } + + @Override + public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOException { + // We do no type conversion here, since that will be done in the ValueSourceReaderOperator for row-stride cases + // Using the BlockLoader.convert(Block) function defined above + return delegate.rowStrideReader(context); + } + + @Override + public StoredFieldsSpec rowStrideStoredFieldSpec() { + return delegate.rowStrideStoredFieldSpec(); + } + + @Override + public boolean supportsOrdinals() { + return delegate.supportsOrdinals(); + } + + @Override + public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException { + return delegate.ordinals(context); + } + + @Override + public final String toString() { + return "TypeConvertingBlockLoader[delegate=" + delegate + ", convertEvaluator=" + convertEvaluator + "]"; + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java index 228ed6c5b4b32..fc00f5be22624 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlPlugin.java @@ -57,21 +57,20 @@ import org.elasticsearch.xpack.esql.action.RestEsqlQueryAction; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; -import org.elasticsearch.xpack.esql.core.index.IndexResolver; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.enrich.EnrichLookupOperator; import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.querydsl.query.SingleValueQuery; -import org.elasticsearch.xpack.esql.session.EsqlIndexResolver; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; +import org.elasticsearch.xpack.esql.type.MultiTypeEsField; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Objects; -import java.util.Set; import java.util.function.Predicate; import java.util.function.Supplier; @@ -110,15 +109,7 @@ public Collection createComponents(PluginServices services) { BlockFactory blockFactory = new BlockFactory(circuitBreaker, bigArrays, maxPrimitiveArrayBlockSize); setupSharedSecrets(); return List.of( - new PlanExecutor( - new IndexResolver( - services.client(), - services.clusterService().getClusterName().value(), - EsqlDataTypeRegistry.INSTANCE, - Set::of - ), - new EsqlIndexResolver(services.client(), EsqlDataTypeRegistry.INSTANCE) - ), + new PlanExecutor(new IndexResolver(services.client(), EsqlDataTypeRegistry.INSTANCE)), new ExchangeService(services.clusterService().getSettings(), services.threadPool(), ThreadPool.Names.SEARCH, blockFactory), blockFactory ); @@ -198,6 +189,7 @@ public List getNamedWriteables() { entries.add(UnsupportedAttribute.ENTRY); // TODO combine with above once these are in the same project entries.addAll(NamedExpression.getNamedWriteables()); entries.add(UnsupportedAttribute.NAMED_EXPRESSION_ENTRY); // TODO combine with above once these are in the same project + entries.add(MultiTypeEsField.ENTRY); // TODO combine with EsField.getNamedWriteables() once these are in the same module return entries; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlStatsRequest.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlStatsRequest.java index 2a0a148459250..1637bcc335ad3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlStatsRequest.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/EsqlStatsRequest.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.plugin; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -34,11 +33,6 @@ public void includeStats(boolean includeStats) { this.includeStats = includeStats; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - @Override public String toString() { return "esql_stats"; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java index 09b2beaa53846..223cdf6f3c9be 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plugin/TransportEsqlStatsAction.java @@ -61,13 +61,13 @@ public TransportEsqlStatsAction( } @Override - protected void resolveRequest(EsqlStatsRequest request, ClusterState clusterState) { + protected DiscoveryNode[] resolveRequest(EsqlStatsRequest request, ClusterState clusterState) { if (featureService.clusterHasFeature(clusterState, ESQL_STATS_FEATURE)) { // use the whole cluster - super.resolveRequest(request, clusterState); + return super.resolveRequest(request, clusterState); } else { // not all nodes in the cluster have upgraded to esql - just use this node for now - request.setConcreteNodes(new DiscoveryNode[] { clusterService.localNode() }); + return new DiscoveryNode[] { clusterService.localNode() }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 1f5374b73466e..0589424b37d1e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -11,7 +11,6 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.common.Strings; import org.elasticsearch.common.regex.Regex; -import org.elasticsearch.core.Assertions; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.logging.LogManager; import org.elasticsearch.logging.Logger; @@ -31,12 +30,9 @@ import org.elasticsearch.xpack.esql.core.expression.UnresolvedStar; import org.elasticsearch.xpack.esql.core.expression.function.FunctionRegistry; import org.elasticsearch.xpack.esql.core.index.IndexResolution; -import org.elasticsearch.xpack.esql.core.index.IndexResolver; import org.elasticsearch.xpack.esql.core.index.MappingException; import org.elasticsearch.xpack.esql.core.plan.TableIdentifier; import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; -import org.elasticsearch.xpack.esql.core.type.DataType; -import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; @@ -59,7 +55,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -68,7 +63,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.xpack.esql.core.index.IndexResolver.UNMAPPED; import static org.elasticsearch.xpack.esql.core.util.ActionListeners.map; import static org.elasticsearch.xpack.esql.core.util.StringUtils.WILDCARD; @@ -79,7 +73,6 @@ public class EsqlSession { private final String sessionId; private final EsqlConfiguration configuration; private final IndexResolver indexResolver; - private final EsqlIndexResolver esqlIndexResolver; private final EnrichPolicyResolver enrichPolicyResolver; private final PreAnalyzer preAnalyzer; @@ -94,7 +87,6 @@ public EsqlSession( String sessionId, EsqlConfiguration configuration, IndexResolver indexResolver, - EsqlIndexResolver esqlIndexResolver, EnrichPolicyResolver enrichPolicyResolver, PreAnalyzer preAnalyzer, FunctionRegistry functionRegistry, @@ -105,7 +97,6 @@ public EsqlSession( this.sessionId = sessionId; this.configuration = configuration; this.indexResolver = indexResolver; - this.esqlIndexResolver = esqlIndexResolver; this.enrichPolicyResolver = enrichPolicyResolver; this.preAnalyzer = preAnalyzer; this.verifier = verifier; @@ -207,12 +198,7 @@ private void preAnalyzeIndices(LogicalPlan parsed, ActionListener void preAnalyzeIndices(LogicalPlan parsed, ActionListener fieldNames, - ActionListener listener - ) { - indexResolver.resolveAsMergedMapping(indexWildcard, fieldNames, false, Map.of(), new ActionListener<>() { - @Override - public void onResponse(IndexResolution fromQl) { - esqlIndexResolver.resolveAsMergedMapping(indexWildcard, fieldNames, new ActionListener<>() { - @Override - public void onResponse(IndexResolution fromEsql) { - if (fromQl.isValid() == false) { - if (fromEsql.isValid()) { - throw new IllegalArgumentException( - "ql and esql didn't make the same resolution: validity differs " + fromQl + " != " + fromEsql - ); - } - } else { - assertSameMappings("", fromQl.get().mapping(), fromEsql.get().mapping()); - if (fromQl.get().concreteIndices().equals(fromEsql.get().concreteIndices()) == false) { - throw new IllegalArgumentException( - "ql and esql didn't make the same resolution: concrete indices differ " - + fromQl.get().concreteIndices() - + " != " - + fromEsql.get().concreteIndices() - ); - } - } - listener.onResponse(fromEsql); - } - - private void assertSameMappings(String prefix, Map fromQl, Map fromEsql) { - List qlFields = new ArrayList<>(); - qlFields.addAll(fromQl.keySet()); - Collections.sort(qlFields); - - List esqlFields = new ArrayList<>(); - esqlFields.addAll(fromEsql.keySet()); - Collections.sort(esqlFields); - if (qlFields.equals(esqlFields) == false) { - throw new IllegalArgumentException( - prefix + ": ql and esql didn't make the same resolution: fields differ \n" + qlFields + " !=\n" + esqlFields - ); - } - - for (int f = 0; f < qlFields.size(); f++) { - String name = qlFields.get(f); - EsField qlField = fromQl.get(name); - EsField esqlField = fromEsql.get(name); - - if (qlField.getProperties().isEmpty() == false || esqlField.getProperties().isEmpty() == false) { - assertSameMappings( - prefix.equals("") ? name : prefix + "." + name, - qlField.getProperties(), - esqlField.getProperties() - ); - } - - /* - * Check that the field itself is the same, skipping isAlias because - * we don't actually use it in ESQL and the EsqlIndexResolver doesn't - * produce exactly the same result. - */ - if (qlField.getDataType().equals(DataType.UNSUPPORTED) == false - && qlField.getName().equals(esqlField.getName()) == false - // QL uses full paths for unsupported fields. ESQL does not. This particular difference is fine. - ) { - throw new IllegalArgumentException( - prefix - + "." - + name - + ": ql and esql didn't make the same resolution: names differ [" - + qlField.getName() - + "] != [" - + esqlField.getName() - + "]" - ); - } - if (qlField.getDataType() != esqlField.getDataType()) { - throw new IllegalArgumentException( - prefix - + "." - + name - + ": ql and esql didn't make the same resolution: types differ [" - + qlField.getDataType() - + "] != [" - + esqlField.getDataType() - + "]" - ); - } - if (qlField.isAggregatable() != esqlField.isAggregatable()) { - throw new IllegalArgumentException( - prefix - + "." - + name - + ": ql and esql didn't make the same resolution: aggregability differ [" - + qlField.isAggregatable() - + "] != [" - + esqlField.isAggregatable() - + "]" - ); - } - } - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }, - EsqlSession::specificValidity, - IndexResolver.PRESERVE_PROPERTIES, - // TODO no matter what metadata fields are asked in a query, the "allowedMetadataFields" is always _index, does it make - // sense to reflect the actual list of metadata fields instead? - IndexResolver.INDEX_METADATA_FIELD - ); - } - static Set fieldNames(LogicalPlan parsed, Set enrichPolicyMatchFields) { if (false == parsed.anyMatch(plan -> plan instanceof Aggregate || plan instanceof Project)) { // no explicit columns selection, for example "from employees" @@ -476,14 +332,14 @@ public void optimizedPhysicalPlan(LogicalPlan logicalPlan, ActionListener types) { - boolean hasUnmapped = types.containsKey(UNMAPPED); + boolean hasUnmapped = types.containsKey(IndexResolver.UNMAPPED); boolean hasTypeConflicts = types.size() > (hasUnmapped ? 2 : 1); String metricConflictsTypeName = null; boolean hasMetricConflicts = false; if (hasTypeConflicts == false) { for (Map.Entry type : types.entrySet()) { - if (UNMAPPED.equals(type.getKey())) { + if (IndexResolver.UNMAPPED.equals(type.getKey())) { continue; } if (type.getValue().metricConflictsIndices() != null && type.getValue().metricConflictsIndices().length > 0) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlIndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java similarity index 89% rename from x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlIndexResolver.java rename to x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index f973983e47f39..5fd7f0c230463 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlIndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -11,13 +11,13 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.Strings; import org.elasticsearch.index.mapper.TimeSeriesParams; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.esql.core.index.EsIndex; import org.elasticsearch.xpack.esql.core.index.IndexResolution; -import org.elasticsearch.xpack.esql.core.index.IndexResolver; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.DataTypeRegistry; import org.elasticsearch.xpack.esql.core.type.DateEsField; @@ -43,11 +43,30 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; -public class EsqlIndexResolver { +public class IndexResolver { + public static final Set ALL_FIELDS = Set.of("*"); + public static final Set INDEX_METADATA_FIELD = Set.of("_index"); + public static final String UNMAPPED = "unmapped"; + + public static final IndicesOptions FIELD_CAPS_INDICES_OPTIONS = IndicesOptions.builder() + .concreteTargetOptions(IndicesOptions.ConcreteTargetOptions.ALLOW_UNAVAILABLE_TARGETS) + .wildcardOptions( + IndicesOptions.WildcardOptions.builder() + .matchOpen(true) + .matchClosed(false) + .includeHidden(false) + .allowEmptyExpressions(true) + .resolveAliases(true) + ) + .gatekeeperOptions( + IndicesOptions.GatekeeperOptions.builder().ignoreThrottled(true).allowClosedIndices(true).allowAliasToMultipleIndices(true) + ) + .build(); + private final Client client; private final DataTypeRegistry typeRegistry; - public EsqlIndexResolver(Client client, DataTypeRegistry typeRegistry) { + public IndexResolver(Client client, DataTypeRegistry typeRegistry) { this.client = client; this.typeRegistry = typeRegistry; } @@ -206,26 +225,10 @@ private EsField conflictingTypes(String name, String fullName, FieldCapabilities if (type == UNSUPPORTED) { return unsupported(name, fc); } - typesToIndices.computeIfAbsent(type.esType(), _key -> new TreeSet<>()).add(ir.getIndexName()); - } - } - StringBuilder errorMessage = new StringBuilder(); - errorMessage.append("mapped as ["); - errorMessage.append(typesToIndices.size()); - errorMessage.append("] incompatible types: "); - boolean first = true; - for (Map.Entry> e : typesToIndices.entrySet()) { - if (first) { - first = false; - } else { - errorMessage.append(", "); + typesToIndices.computeIfAbsent(type.typeName(), _key -> new TreeSet<>()).add(ir.getIndexName()); } - errorMessage.append("["); - errorMessage.append(e.getKey()); - errorMessage.append("] in "); - errorMessage.append(e.getValue()); } - return new InvalidMappedField(name, errorMessage.toString()); + return new InvalidMappedField(name, typesToIndices); } private EsField conflictingMetricTypes(String name, String fullName, FieldCapabilitiesResponse fieldCapsResponse) { @@ -245,7 +248,7 @@ private static FieldCapabilitiesRequest createFieldCapsRequest(String index, Set req.includeUnmapped(true); // lenient because we throw our own errors looking at the response e.g. if something was not resolved // also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable - req.indicesOptions(IndexResolver.FIELD_CAPS_INDICES_OPTIONS); + req.indicesOptions(FIELD_CAPS_INDICES_OPTIONS); req.setMergeResults(false); return req; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index cc2525799224b..23a94bde56b1e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -211,7 +211,7 @@ public static TemporalAmount parseTemporalAmount(Object val, DataType expectedTy * Throws QlIllegalArgumentException if such conversion is not possible */ public static Object convert(Object value, DataType dataType) { - DataType detectedType = EsqlDataTypes.fromJava(value); + DataType detectedType = DataType.fromJava(value); if (detectedType == dataType || value == null) { return value; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java index dc680e5305842..ee28a7fe9941a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistry.java @@ -44,7 +44,7 @@ public DataType fromEs(String typeName, TimeSeriesParams.MetricType metricType) @Override public DataType fromJava(Object value) { - return EsqlDataTypes.fromJava(value); + return DataType.fromJava(value); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java index e48b46758f36c..2d817d65f6ba9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypes.java @@ -6,17 +6,14 @@ */ package org.elasticsearch.xpack.esql.type; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.xpack.esql.core.type.DataType; import java.util.Collections; import java.util.Locale; import java.util.Map; -import java.util.function.Function; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toUnmodifiableMap; -import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; @@ -52,15 +49,6 @@ public final class EsqlDataTypes { ES_TO_TYPE = Collections.unmodifiableMap(map); } - private static final Map NAME_OR_ALIAS_TO_TYPE; - static { - Map map = DataType.types().stream().collect(toMap(DataType::typeName, Function.identity())); - map.put("bool", BOOLEAN); - map.put("int", INTEGER); - map.put("string", KEYWORD); - NAME_OR_ALIAS_TO_TYPE = Collections.unmodifiableMap(map); - } - private EsqlDataTypes() {} public static DataType fromTypeName(String name) { @@ -72,37 +60,6 @@ public static DataType fromName(String name) { return type != null ? type : UNSUPPORTED; } - public static DataType fromNameOrAlias(String typeName) { - DataType type = NAME_OR_ALIAS_TO_TYPE.get(typeName.toLowerCase(Locale.ROOT)); - return type != null ? type : UNSUPPORTED; - } - - public static DataType fromJava(Object value) { - if (value == null) { - return NULL; - } - if (value instanceof Boolean) { - return BOOLEAN; - } - if (value instanceof Integer) { - return INTEGER; - } - if (value instanceof Long) { - return LONG; - } - if (value instanceof Double) { - return DOUBLE; - } - if (value instanceof Float) { - return FLOAT; - } - if (value instanceof String || value instanceof Character || value instanceof BytesRef) { - return KEYWORD; - } - - return null; - } - public static boolean isUnsupported(DataType type) { return DataType.isUnsupported(type); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/MultiTypeEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/MultiTypeEsField.java new file mode 100644 index 0000000000000..2b963e7428e2b --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/MultiTypeEsField.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.type; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * During IndexResolution it could occur that the same field is mapped to different types in different indices. + * The class MultiTypeEfField.UnresolvedField holds that information and allows for later resolution of the field + * to a single type during LogicalPlanOptimization. + * If the plan contains conversion expressions for the different types, the resolution will be done using the conversion expressions, + * in which case a MultiTypeEsField will be created to encapsulate the type resolution capabilities. + * This class can be communicated to the data nodes and used during physical planning to influence field extraction so that + * type conversion is done at the data node level. + */ +public class MultiTypeEsField extends EsField { + public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( + EsField.class, + "MultiTypeEsField", + MultiTypeEsField::new + ); + + private final Map indexToConversionExpressions; + + public MultiTypeEsField(String name, DataType dataType, boolean aggregatable, Map indexToConversionExpressions) { + super(name, dataType, Map.of(), aggregatable); + this.indexToConversionExpressions = indexToConversionExpressions; + } + + public MultiTypeEsField(StreamInput in) throws IOException { + // TODO: Change the conversion expression serialization to i.readNamedWriteable(Expression.class) once Expression is fully supported + this(in.readString(), DataType.readFrom(in), in.readBoolean(), in.readImmutableMap(i -> ((PlanStreamInput) i).readExpression())); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(getName()); + out.writeString(getDataType().typeName()); + out.writeBoolean(isAggregatable()); + out.writeMap(getIndexToConversionExpressions(), (o, v) -> out.writeNamedWriteable(v)); + } + + @Override + public String getWriteableName() { + return ENTRY.name; + } + + public Map getIndexToConversionExpressions() { + return indexToConversionExpressions; + } + + public Expression getConversionExpressionForIndex(String indexName) { + return indexToConversionExpressions.get(indexName); + } + + public static MultiTypeEsField resolveFrom( + InvalidMappedField invalidMappedField, + Map typesToConversionExpressions + ) { + Map> typesToIndices = invalidMappedField.getTypesToIndices(); + DataType resolvedDataType = DataType.UNSUPPORTED; + Map indexToConversionExpressions = new HashMap<>(); + for (String typeName : typesToIndices.keySet()) { + Set indices = typesToIndices.get(typeName); + Expression convertExpr = typesToConversionExpressions.get(typeName); + if (resolvedDataType == DataType.UNSUPPORTED) { + resolvedDataType = convertExpr.dataType(); + } else if (resolvedDataType != convertExpr.dataType()) { + throw new IllegalArgumentException("Resolved data type mismatch: " + resolvedDataType + " != " + convertExpr.dataType()); + } + for (String indexName : indices) { + indexToConversionExpressions.put(indexName, convertExpr); + } + } + return new MultiTypeEsField(invalidMappedField.getName(), resolvedDataType, false, indexToConversionExpressions); + } + + @Override + public boolean equals(Object obj) { + if (super.equals(obj) == false) { + return false; + } + if (obj instanceof MultiTypeEsField other) { + return super.equals(other) && indexToConversionExpressions.equals(other.indexToConversionExpressions); + } + return false; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), indexToConversionExpressions); + } + + @Override + public String toString() { + return super.toString() + " (" + indexToConversionExpressions + ")"; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 44466cebb7dac..27aa985efd6d0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -222,6 +222,14 @@ public CsvTests(String fileName, String groupName, String testName, Integer line public final void test() throws Throwable { try { assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, Version.CURRENT)); + /* + * The csv tests support all but a few features. The unsupported features + * are tested in integration tests. + */ + assumeFalse("metadata fields aren't supported", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METADATA_FIELDS))); + assumeFalse("enrich can't load fields in csv tests", testCase.requiredCapabilities.contains(cap(EsqlFeatures.ENRICH_LOAD))); + assumeFalse("can't load metrics in csv tests", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METRICS_SYNTAX))); + assumeFalse("multiple indices aren't supported", testCase.requiredCapabilities.contains(EsqlCapabilities.UNION_TYPES)); if (Build.current().isSnapshot()) { assertThat( @@ -231,14 +239,6 @@ public final void test() throws Throwable { ); } - /* - * The csv tests support all but a few features. The unsupported features - * are tested in integration tests. - */ - assumeFalse("metadata fields aren't supported", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METADATA_FIELDS))); - assumeFalse("enrich can't load fields in csv tests", testCase.requiredCapabilities.contains(cap(EsqlFeatures.ENRICH_LOAD))); - assumeFalse("can't load metrics in csv tests", testCase.requiredCapabilities.contains(cap(EsqlFeatures.METRICS_SYNTAX))); - doTest(); } catch (Throwable th) { throw reworkException(th); @@ -334,7 +334,7 @@ private PhysicalPlan physicalPlan(LogicalPlan parsed, CsvTestsDataLoader.TestsDa private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed) { var preAnalysis = new PreAnalyzer().preAnalyze(parsed); var indices = preAnalysis.indices; - if (indices.size() == 0) { + if (indices.isEmpty()) { /* * If the data set doesn't matter we'll just grab one we know works. * Employees is fine. @@ -345,11 +345,23 @@ private static CsvTestsDataLoader.TestsDataset testsDataset(LogicalPlan parsed) } String indexName = indices.get(0).id().index(); - var dataset = CSV_DATASET_MAP.get(indexName); - if (dataset == null) { + List datasets = new ArrayList<>(); + if (indexName.endsWith("*")) { + String indexPrefix = indexName.substring(0, indexName.length() - 1); + for (var entry : CSV_DATASET_MAP.entrySet()) { + if (entry.getKey().startsWith(indexPrefix)) { + datasets.add(entry.getValue()); + } + } + } else { + var dataset = CSV_DATASET_MAP.get(indexName); + datasets.add(dataset); + } + if (datasets.isEmpty()) { throw new IllegalArgumentException("unknown CSV dataset for table [" + indexName + "]"); } - return dataset; + // TODO: Support multiple datasets + return datasets.get(0); } private static TestPhysicalOperationProviders testOperationProviders(CsvTestsDataLoader.TestsDataset dataset) throws Exception { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 975d8e1c7d7b8..794bdc23f08c5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -53,7 +53,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.local.EsqlProject; import org.elasticsearch.xpack.esql.plugin.EsqlPlugin; -import org.elasticsearch.xpack.esql.session.EsqlIndexResolver; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import java.io.IOException; @@ -2106,7 +2106,7 @@ protected List filteredWarnings() { private static LogicalPlan analyzeWithEmptyFieldCapsResponse(String query) throws IOException { List idxResponses = List.of(new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true)); FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); - IndexResolution resolution = new EsqlIndexResolver(null, EsqlDataTypeRegistry.INSTANCE).mergedMappings("test*", caps); + IndexResolution resolution = new IndexResolver(null, EsqlDataTypeRegistry.INSTANCE).mergedMappings("test*", caps); var analyzer = analyzer(resolution, TEST_VERIFIER, configuration(query)); return analyze(query, analyzer); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java index 223ee08316479..27a42f79e39ff 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/ParsingTests.java @@ -7,17 +7,36 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.core.PathUtils; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.esql.core.ParsingException; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.function.FunctionDefinition; import org.elasticsearch.xpack.esql.core.index.EsIndex; import org.elasticsearch.xpack.esql.core.index.IndexResolution; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.TypesTests; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.parser.EsqlParser; +import org.elasticsearch.xpack.esql.plan.logical.Row; +import org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.esql.EsqlTestUtils.TEST_VERIFIER; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; public class ParsingTests extends ESTestCase { private static final String INDEX_NAME = "test"; @@ -53,6 +72,49 @@ public void testLeastFunctionInvalidInputs() { assertEquals("1:23: error building [least]: expects at least one argument", error("row a = 1 | eval x = least()")); } + /** + * Tests the inline cast syntax {@code ::} for all supported types and + * builds a little json report of the valid types. + */ + public void testInlineCast() throws IOException { + EsqlFunctionRegistry registry = new EsqlFunctionRegistry(); + Path dir = PathUtils.get(System.getProperty("java.io.tmpdir")).resolve("esql").resolve("functions").resolve("kibana"); + Files.createDirectories(dir); + Path file = dir.resolve("inline_cast.json"); + try (XContentBuilder report = new XContentBuilder(JsonXContent.jsonXContent, Files.newOutputStream(file))) { + report.humanReadable(true).prettyPrint(); + report.startObject(); + List namesAndAliases = new ArrayList<>(DataType.namesAndAliases()); + Collections.sort(namesAndAliases); + for (String nameOrAlias : namesAndAliases) { + DataType expectedType = DataType.fromNameOrAlias(nameOrAlias); + if (expectedType == DataType.TEXT) { + expectedType = DataType.KEYWORD; + } + if (EsqlDataTypeConverter.converterFunctionFactory(expectedType) == null) { + continue; + } + LogicalPlan plan = parser.createStatement("ROW a = 1::" + nameOrAlias); + Row row = as(plan, Row.class); + assertThat(row.fields(), hasSize(1)); + Expression functionCall = row.fields().get(0).child(); + assertThat(functionCall.dataType(), equalTo(expectedType)); + report.field(nameOrAlias, functionName(registry, functionCall)); + } + report.endObject(); + } + logger.info("Wrote to file: {}", file); + } + + private String functionName(EsqlFunctionRegistry registry, Expression functionCall) { + for (FunctionDefinition def : registry.listFunctions()) { + if (functionCall.getClass().equals(def.clazz())) { + return def.name(); + } + } + throw new IllegalArgumentException("can't find name for " + functionCall); + } + private String error(String query) { ParsingException e = expectThrows(ParsingException.class, () -> defaultAnalyzer.analyze(parser.createStatement(query))); String message = e.getMessage(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index e5f59f1ffa8ad..8eef05bd9687b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -564,7 +564,7 @@ private String error(String query, Analyzer analyzer, Object... params) { } else if (param instanceof String) { parameters.add(new QueryParam(null, param, KEYWORD)); } else if (param instanceof Number) { - parameters.add(new QueryParam(null, param, EsqlDataTypes.fromJava(param))); + parameters.add(new QueryParam(null, param, DataType.fromJava(param))); } else { throw new IllegalArgumentException("VerifierTests don't support params of type " + param.getClass()); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java index 90fca14b7b06d..9f81437bd1b77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/enrich/EnrichPolicyResolverTests.java @@ -11,10 +11,12 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.ActionType; -import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.client.internal.FilterClient; import org.elasticsearch.cluster.ClusterName; @@ -36,8 +38,8 @@ import org.elasticsearch.xpack.core.enrich.EnrichMetadata; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; -import org.elasticsearch.xpack.esql.core.index.IndexResolver; import org.elasticsearch.xpack.esql.plan.logical.Enrich; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import org.junit.After; import org.junit.Before; @@ -416,7 +418,7 @@ class TestEnrichPolicyResolver extends EnrichPolicyResolver { super( mockClusterService(policies), transports.get(cluster), - new IndexResolver(new FieldCapsClient(threadPool, aliases, mappings), cluster, EsqlDataTypeRegistry.INSTANCE, Set::of) + new IndexResolver(new FieldCapsClient(threadPool, aliases, mappings), EsqlDataTypeRegistry.INSTANCE) ); this.policies = policies; this.cluster = cluster; @@ -483,30 +485,19 @@ protected void String alias = aliases.get(r.indices()[0]); assertNotNull(alias); Map mapping = mappings.get(alias); + final FieldCapabilitiesResponse response; if (mapping != null) { - Map> fieldCaps = new HashMap<>(); + Map fieldCaps = new HashMap<>(); for (Map.Entry e : mapping.entrySet()) { - var f = new FieldCapabilities( - e.getKey(), - e.getValue(), - false, - false, - false, - true, - null, - new String[] { alias }, - null, - null, - null, - null, - Map.of() - ); - fieldCaps.put(e.getKey(), Map.of(e.getValue(), f)); + var f = new IndexFieldCapabilities(e.getKey(), e.getValue(), false, false, false, false, null, Map.of()); + fieldCaps.put(e.getKey(), f); } - listener.onResponse((Response) new FieldCapabilitiesResponse(new String[] { alias }, fieldCaps)); + var indexResponse = new FieldCapabilitiesIndexResponse(alias, null, fieldCaps, true); + response = new FieldCapabilitiesResponse(List.of(indexResponse), List.of()); } else { - listener.onResponse((Response) new FieldCapabilitiesResponse(new String[0], Map.of())); + response = new FieldCapabilitiesResponse(List.of(), List.of()); } + threadPool().executor(ThreadPool.Names.SEARCH_COORDINATION).execute(ActionRunnable.supply(listener, () -> (Response) response)); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java index a5ce5e004b194..33f9cb3123b8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractExpressionSerializationTests.java @@ -22,17 +22,23 @@ import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests; +import org.junit.Before; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; public abstract class AbstractExpressionSerializationTests extends AbstractWireTestCase { + /** + * We use a single random config for all serialization because it's pretty + * heavy to build, especially in {@link #testConcurrentSerialization()}. + */ + private EsqlConfiguration config; + public static Source randomSource() { int lineNumber = between(0, EXAMPLE_QUERY.length - 1); int offset = between(0, EXAMPLE_QUERY[lineNumber].length() - 2); @@ -47,10 +53,6 @@ public static Expression randomChild() { @Override protected final T copyInstance(T instance, TransportVersion version) throws IOException { - EsqlConfiguration config = EsqlConfigurationSerializationTests.randomConfiguration( - Arrays.stream(EXAMPLE_QUERY).collect(Collectors.joining("\n")), - Map.of() - ); return copyInstance( instance, getNamedWriteableRegistry(), @@ -59,15 +61,27 @@ protected final T copyInstance(T instance, TransportVersion version) throws IOEx PlanStreamInput pin = new PlanStreamInput(in, new PlanNameRegistry(), in.namedWriteableRegistry(), config); @SuppressWarnings("unchecked") T deser = (T) pin.readNamedWriteable(Expression.class); - assertThat(deser.source(), equalTo(instance.source())); + if (alwaysEmptySource()) { + assertThat(deser.source(), sameInstance(Source.EMPTY)); + } else { + assertThat(deser.source(), equalTo(instance.source())); + } return deser; }, version ); } + protected boolean alwaysEmptySource() { + return false; + } + protected abstract List getNamedWriteables(); + public EsqlConfiguration configuration() { + return config; + } + @Override protected final NamedWriteableRegistry getNamedWriteableRegistry() { List entries = new ArrayList<>(NamedExpression.getNamedWriteables()); @@ -87,4 +101,9 @@ protected final NamedWriteableRegistry getNamedWriteableRegistry() { "I understand equations, both the simple and quadratical,", "About binomial theorem I'm teeming with a lot o' news,", "With many cheerful facts about the square of the hypotenuse." }; + + @Before + public void initConfig() { + config = EsqlConfigurationSerializationTests.randomConfiguration(String.join("\n", EXAMPLE_QUERY), Map.of()); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractVarargsSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractVarargsSerializationTests.java new file mode 100644 index 0000000000000..67195fa99114b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/AbstractVarargsSerializationTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.io.IOException; +import java.util.List; + +public abstract class AbstractVarargsSerializationTests extends AbstractExpressionSerializationTests { + protected abstract T create(Source source, Expression first, List rest); + + @Override + protected final T createTestInstance() { + Source source = randomSource(); + Expression first = randomChild(); + List rest = randomList(0, 10, AbstractExpressionSerializationTests::randomChild); + return create(source, first, rest); + } + + @Override + protected final T mutateInstance(T instance) throws IOException { + Source source = instance.source(); + Expression first = instance.children().get(0); + List rest = instance.children().subList(1, instance.children().size()); + if (randomBoolean()) { + first = randomValueOtherThan(first, AbstractExpressionSerializationTests::randomChild); + } else { + rest = randomValueOtherThan(rest, () -> randomList(0, 10, AbstractExpressionSerializationTests::randomChild)); + } + return create(instance.source(), first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/LiteralSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/LiteralSerializationTests.java new file mode 100644 index 0000000000000..39e18bf9761ec --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/LiteralSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.LiteralTests; + +import java.io.IOException; +import java.util.List; + +public class LiteralSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected Literal createTestInstance() { + return LiteralTests.randomLiteral(); + } + + @Override + protected Literal mutateInstance(Literal instance) throws IOException { + return LiteralTests.mutateLiteral(instance); + } + + @Override + protected List getNamedWriteables() { + return List.of(Literal.ENTRY); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/OrderSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/OrderSerializationTests.java new file mode 100644 index 0000000000000..dd2671f4cf86d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/OrderSerializationTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.io.IOException; +import java.util.List; + +public class OrderSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected Order createTestInstance() { + return new Order(randomSource(), randomChild(), randomDirection(), randomNulls()); + } + + private static org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection randomDirection() { + return randomFrom(org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection.values()); + } + + private static org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition randomNulls() { + return randomFrom(org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition.values()); + } + + @Override + protected Order mutateInstance(Order instance) throws IOException { + Source source = instance.source(); + Expression child = instance.child(); + org.elasticsearch.xpack.esql.core.expression.Order.OrderDirection direction = instance.direction(); + org.elasticsearch.xpack.esql.core.expression.Order.NullsPosition nulls = instance.nullsPosition(); + switch (between(0, 2)) { + case 0 -> child = randomValueOtherThan(child, AbstractExpressionSerializationTests::randomChild); + case 1 -> direction = randomValueOtherThan(direction, OrderSerializationTests::randomDirection); + case 2 -> nulls = randomValueOtherThan(nulls, OrderSerializationTests::randomNulls); + } + return new Order(source, child, direction, nulls); + } + + @Override + protected List getNamedWriteables() { + return List.of(Order.ENTRY); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 20c2b6df9710a..f27438de6df6b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -641,6 +641,46 @@ public interface ExpectedEvaluatorToString { Matcher evaluatorToString(int nullPosition, TestCaseSupplier.TypedData nullData, Matcher original); } + /** + * Modifies suppliers to generate BytesRefs with random offsets. + */ + protected static List randomizeBytesRefsOffset(List testCaseSuppliers) { + return testCaseSuppliers.stream().map(supplier -> new TestCaseSupplier(supplier.name(), supplier.types(), () -> { + var testCase = supplier.supplier().get(); + + var newData = testCase.getData().stream().map(typedData -> { + if (typedData.data() instanceof BytesRef bytesRef) { + var offset = randomIntBetween(0, 10); + var extraLength = randomIntBetween(0, 10); + var newBytesArray = randomByteArrayOfLength(bytesRef.length + offset + extraLength); + + System.arraycopy(bytesRef.bytes, bytesRef.offset, newBytesArray, offset, bytesRef.length); + + var newBytesRef = new BytesRef(newBytesArray, offset, bytesRef.length); + var newTypedData = new TestCaseSupplier.TypedData(newBytesRef, typedData.type(), typedData.name()); + + if (typedData.isForceLiteral()) { + newTypedData.forceLiteral(); + } + + return newTypedData; + } + return typedData; + }).toList(); + + return new TestCaseSupplier.TestCase( + newData, + testCase.evaluatorToString(), + testCase.expectedType(), + testCase.getMatcher(), + testCase.getExpectedWarnings(), + testCase.getExpectedTypeError(), + testCase.foldingExceptionClass(), + testCase.foldingExceptionMessage() + ); + })).toList(); + } + protected static List anyNullIsNull( List testCaseSuppliers, ExpectedType expectedType, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 54c4f2ae07eca..7eadad58ec09b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -1417,7 +1417,7 @@ public TypedData(Object data, DataType type, String name) { * @param name a name for the value, used for generating test case names */ public TypedData(Object data, String name) { - this(data, EsqlDataTypes.fromJava(data), name); + this(data, DataType.fromJava(data), name); } /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NotSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NotSerializationTests.java new file mode 100644 index 0000000000000..61e3690f1633f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/NotSerializationTests.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; +import java.util.List; + +public class NotSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return UnaryScalarFunction.getNamedWriteables(); + } + + @Override + protected Not createTestInstance() { + return new Not(randomSource(), randomChild()); + } + + @Override + protected Not mutateInstance(Not instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new Not(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseSerializationTests.java new file mode 100644 index 0000000000000..69bbf2f76937f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class CaseSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Case create(Source source, Expression first, List rest) { + return new Case(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index f24955eb4804a..02da8ea22a6a0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.elasticsearch.xpack.esql.type.EsqlDataTypes; import java.math.BigInteger; import java.util.List; @@ -334,7 +333,7 @@ private static Case caseExpr(Object... args) { if (arg instanceof Expression e) { return e; } - return new Literal(Source.synthetic(arg == null ? "null" : arg.toString()), arg, EsqlDataTypes.fromJava(arg)); + return new Literal(Source.synthetic(arg == null ? "null" : arg.toString()), arg, DataType.fromJava(arg)); }).toList(); return new Case(Source.synthetic(""), exps.get(0), exps.subList(1, exps.size())); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestSerializationTests.java new file mode 100644 index 0000000000000..43e1fe405911a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/GreatestSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class GreatestSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Greatest create(Source source, Expression first, List rest) { + return new Greatest(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastSerializationTests.java new file mode 100644 index 0000000000000..f552713af4dbe --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/LeastSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.conditional; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class LeastSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Least create(Source source, Expression first, List rest) { + return new Least(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffSerializationTests.java new file mode 100644 index 0000000000000..b1dc1b064ae5a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffSerializationTests.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateDiffSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateDiff createTestInstance() { + Source source = randomSource(); + Expression unit = randomChild(); + Expression startTimestamp = randomChild(); + Expression endTimestamp = randomChild(); + return new DateDiff(source, unit, startTimestamp, endTimestamp); + } + + @Override + protected DateDiff mutateInstance(DateDiff instance) throws IOException { + Source source = instance.source(); + Expression unit = instance.unit(); + Expression startTimestamp = instance.startTimestamp(); + Expression endTimestamp = instance.endTimestamp(); + switch (between(0, 2)) { + case 0 -> unit = randomValueOtherThan(unit, AbstractExpressionSerializationTests::randomChild); + case 1 -> startTimestamp = randomValueOtherThan(startTimestamp, AbstractExpressionSerializationTests::randomChild); + case 2 -> endTimestamp = randomValueOtherThan(endTimestamp, AbstractExpressionSerializationTests::randomChild); + } + return new DateDiff(source, unit, startTimestamp, endTimestamp); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractSerializationTests.java new file mode 100644 index 0000000000000..6e1c061c84f2e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateExtractSerializationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateExtractSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateExtract createTestInstance() { + Source source = randomSource(); + Expression datePart = randomChild(); + Expression field = randomChild(); + return new DateExtract(source, datePart, field, configuration()); + } + + @Override + protected DateExtract mutateInstance(DateExtract instance) throws IOException { + Source source = instance.source(); + Expression datePart = instance.datePart(); + Expression field = instance.field(); + if (randomBoolean()) { + datePart = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } else { + field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } + return new DateExtract(source, datePart, field, configuration()); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatSerializationTests.java new file mode 100644 index 0000000000000..4dff735318558 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateFormatSerializationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateFormatSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateFormat createTestInstance() { + Source source = randomSource(); + Expression field = randomChild(); + Expression format = randomBoolean() ? null : randomChild(); + return new DateFormat(source, field, format, configuration()); + } + + @Override + protected DateFormat mutateInstance(DateFormat instance) throws IOException { + Source source = instance.source(); + Expression field = instance.field(); + Expression format = instance.format(); + if (randomBoolean()) { + field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } else { + format = randomValueOtherThan(format, () -> randomBoolean() ? null : randomChild()); + } + return new DateFormat(source, field, format, configuration()); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java new file mode 100644 index 0000000000000..e816f2c4a20fb --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateParseSerializationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateParseSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateParse createTestInstance() { + Source source = randomSource(); + Expression first = randomChild(); + Expression second = randomBoolean() ? null : randomChild(); + return new DateParse(source, first, second); + } + + @Override + protected DateParse mutateInstance(DateParse instance) throws IOException { + Source source = instance.source(); + Expression first = instance.children().get(0); + Expression second = instance.children().size() == 1 ? null : instance.children().get(1); + if (randomBoolean()) { + first = randomValueOtherThan(first, AbstractExpressionSerializationTests::randomChild); + } else { + second = randomValueOtherThan(second, () -> randomBoolean() ? null : randomChild()); + } + return new DateParse(source, first, second); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncSerializationTests.java new file mode 100644 index 0000000000000..09d2e06003128 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTruncSerializationTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class DateTruncSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected DateTrunc createTestInstance() { + Source source = randomSource(); + Expression interval = randomChild(); + Expression field = randomChild(); + return new DateTrunc(source, interval, field); + } + + @Override + protected DateTrunc mutateInstance(DateTrunc instance) throws IOException { + Source source = instance.source(); + Expression interval = instance.interval(); + Expression field = instance.field(); + if (randomBoolean()) { + interval = randomValueOtherThan(interval, AbstractExpressionSerializationTests::randomChild); + } else { + field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } + return new DateTrunc(source, interval, field); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowSerializationTests.java new file mode 100644 index 0000000000000..3bb8c2f260561 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.date; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class NowSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Now createTestInstance() { + return new Now(randomSource(), configuration()); + } + + @Override + protected Now mutateInstance(Now instance) throws IOException { + return null; + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java similarity index 97% rename from x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java rename to x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java index 2c1322abf8cda..8edc21db427d2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/math/NowTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/NowTests.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.esql.expression.function.scalar.math; +package org.elasticsearch.xpack.esql.expression.function.scalar.date; import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; import org.elasticsearch.xpack.esql.expression.function.scalar.AbstractConfigurationFunctionTestCase; -import org.elasticsearch.xpack.esql.expression.function.scalar.date.Now; import org.elasticsearch.xpack.esql.session.EsqlConfiguration; import org.hamcrest.Matcher; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java index 063a057134d7e..d2b5e0a455229 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/ip/IpPrefixTests.java @@ -106,7 +106,7 @@ public static Iterable parameters() { }) ); - return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, suppliers))); + return parameterSuppliersFromTypedData(errorsForCasesWithoutExamples(anyNullIsNull(true, randomizeBytesRefsOffset(suppliers)))); } @Override diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMvSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMvSerializationTests.java new file mode 100644 index 0000000000000..fba33c9ea1c03 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMvSerializationTests.java @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.util.List; + +public abstract class AbstractMvSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return AbstractMultivalueFunction.getNamedWriteables(); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendSerializationTests.java new file mode 100644 index 0000000000000..8afd1b44dc3f3 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendSerializationTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvAppendSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvAppend createTestInstance() { + Source source = randomSource(); + Expression field1 = randomChild(); + Expression field2 = randomChild(); + return new MvAppend(source, field1, field2); + } + + @Override + protected MvAppend mutateInstance(MvAppend instance) throws IOException { + Source source = randomSource(); + Expression field1 = randomChild(); + Expression field2 = randomChild(); + if (randomBoolean()) { + field1 = randomValueOtherThan(field1, AbstractExpressionSerializationTests::randomChild); + } else { + field2 = randomValueOtherThan(field2, AbstractExpressionSerializationTests::randomChild); + } + return new MvAppend(source, field1, field2); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java index 6361360652a87..bc1a64da1cc73 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendTests.java @@ -13,6 +13,18 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geo.ShapeTestUtils; +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -235,8 +247,25 @@ private static void bytesRefs(List suppliers) { })); suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_SHAPE, DataType.GEO_SHAPE), () -> { - List field1 = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean())))); - List field2 = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean())))); + GeometryPointCountVisitor pointCounter = new GeometryPointCountVisitor(); + List field1 = randomList( + 1, + 3, + () -> new BytesRef( + GEO.asWkt( + randomValueOtherThanMany(g -> g.visit(pointCounter) > 500, () -> GeometryTestUtils.randomGeometry(randomBoolean())) + ) + ) + ); + List field2 = randomList( + 1, + 3, + () -> new BytesRef( + GEO.asWkt( + randomValueOtherThanMany(g -> g.visit(pointCounter) > 500, () -> GeometryTestUtils.randomGeometry(randomBoolean())) + ) + ) + ); var result = new ArrayList<>(field1); result.addAll(field2); return new TestCaseSupplier.TestCase( @@ -251,8 +280,25 @@ private static void bytesRefs(List suppliers) { })); suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_SHAPE, DataType.CARTESIAN_SHAPE), () -> { - List field1 = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean())))); - List field2 = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean())))); + GeometryPointCountVisitor pointCounter = new GeometryPointCountVisitor(); + List field1 = randomList( + 1, + 3, + () -> new BytesRef( + GEO.asWkt( + randomValueOtherThanMany(g -> g.visit(pointCounter) > 500, () -> ShapeTestUtils.randomGeometry(randomBoolean())) + ) + ) + ); + List field2 = randomList( + 1, + 3, + () -> new BytesRef( + GEO.asWkt( + randomValueOtherThanMany(g -> g.visit(pointCounter) > 500, () -> ShapeTestUtils.randomGeometry(randomBoolean())) + ) + ) + ); var result = new ArrayList<>(field1); result.addAll(field2); return new TestCaseSupplier.TestCase( @@ -293,4 +339,65 @@ private static void nulls(List suppliers) { ); })); } + + public static class GeometryPointCountVisitor implements GeometryVisitor { + + @Override + public Integer visit(Circle circle) throws RuntimeException { + return 2; + } + + @Override + public Integer visit(GeometryCollection collection) throws RuntimeException { + int size = 0; + for (Geometry geometry : collection) { + size += geometry.visit(this); + } + return size; + } + + @Override + public Integer visit(Line line) throws RuntimeException { + return line.length(); + } + + @Override + public Integer visit(LinearRing ring) throws RuntimeException { + return ring.length(); + } + + @Override + public Integer visit(MultiLine multiLine) throws RuntimeException { + return visit((GeometryCollection) multiLine); + } + + @Override + public Integer visit(MultiPoint multiPoint) throws RuntimeException { + return multiPoint.size(); + } + + @Override + public Integer visit(MultiPolygon multiPolygon) throws RuntimeException { + return visit((GeometryCollection) multiPolygon); + } + + @Override + public Integer visit(Point point) throws RuntimeException { + return 1; + } + + @Override + public Integer visit(Polygon polygon) throws RuntimeException { + int size = polygon.getPolygon().length(); + for (int i = 0; i < polygon.getNumberOfHoles(); i++) { + size += polygon.getHole(i).length(); + } + return size; + } + + @Override + public Integer visit(Rectangle rectangle) throws RuntimeException { + return 4; + } + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgSerializationTests.java new file mode 100644 index 0000000000000..f70702b001492 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAvgSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvAvgSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvAvg createTestInstance() { + return new MvAvg(randomSource(), randomChild()); + } + + @Override + protected MvAvg mutateInstance(MvAvg instance) throws IOException { + return new MvAvg(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatSerializationTests.java new file mode 100644 index 0000000000000..9f2aba8d9d9ca --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvConcatSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvConcatSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvConcat createTestInstance() { + Source source = randomSource(); + Expression left = randomChild(); + Expression right = randomChild(); + return new MvConcat(source, left, right); + } + + @Override + protected MvConcat mutateInstance(MvConcat instance) throws IOException { + Source source = instance.source(); + Expression left = instance.left(); + Expression right = instance.right(); + if (randomBoolean()) { + left = randomValueOtherThan(left, AbstractExpressionSerializationTests::randomChild); + } else { + right = randomValueOtherThan(right, AbstractExpressionSerializationTests::randomChild); + } + return new MvConcat(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountSerializationTests.java new file mode 100644 index 0000000000000..a0d28a6cf925b --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvCountSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvCount createTestInstance() { + return new MvCount(randomSource(), randomChild()); + } + + @Override + protected MvCount mutateInstance(MvCount instance) throws IOException { + return new MvCount(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeSerializationTests.java new file mode 100644 index 0000000000000..afb2ec90e1e3e --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvDedupeSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvDedupe createTestInstance() { + return new MvDedupe(randomSource(), randomChild()); + } + + @Override + protected MvDedupe mutateInstance(MvDedupe instance) throws IOException { + return new MvDedupe(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstSerializationTests.java new file mode 100644 index 0000000000000..dbb49bb96a663 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvFirstSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvFirst createTestInstance() { + return new MvFirst(randomSource(), randomChild()); + } + + @Override + protected MvFirst mutateInstance(MvFirst instance) throws IOException { + return new MvFirst(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastSerializationTests.java new file mode 100644 index 0000000000000..190eb0263c162 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvLastSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvLast createTestInstance() { + return new MvLast(randomSource(), randomChild()); + } + + @Override + protected MvLast mutateInstance(MvLast instance) throws IOException { + return new MvLast(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxSerializationTests.java new file mode 100644 index 0000000000000..ffc51af5f103d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvMaxSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvMax createTestInstance() { + return new MvMax(randomSource(), randomChild()); + } + + @Override + protected MvMax mutateInstance(MvMax instance) throws IOException { + return new MvMax(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianSerializationTests.java new file mode 100644 index 0000000000000..067cc6430ce01 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMedianSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvMedianSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvMedian createTestInstance() { + return new MvMedian(randomSource(), randomChild()); + } + + @Override + protected MvMedian mutateInstance(MvMedian instance) throws IOException { + return new MvMedian(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinSerializationTests.java new file mode 100644 index 0000000000000..1f38587274353 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvMinSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvMin createTestInstance() { + return new MvMin(randomSource(), randomChild()); + } + + @Override + protected MvMin mutateInstance(MvMin instance) throws IOException { + return new MvMin(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceSerializationTests.java new file mode 100644 index 0000000000000..64209ce0f4644 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceSerializationTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvSliceSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvSlice createTestInstance() { + Source source = randomSource(); + Expression field = randomChild(); + Expression start = randomChild(); + Expression end = randomBoolean() ? null : randomChild(); + return new MvSlice(source, field, start, end); + } + + @Override + protected MvSlice mutateInstance(MvSlice instance) throws IOException { + Source source = instance.source(); + Expression field = instance.field(); + Expression start = instance.start(); + Expression end = instance.end(); + switch (between(0, 2)) { + case 0 -> field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + case 1 -> start = randomValueOtherThan(start, AbstractExpressionSerializationTests::randomChild); + case 2 -> end = randomValueOtherThan(end, () -> randomBoolean() ? null : randomChild()); + } + return new MvSlice(source, field, start, end); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java index 3ab17b78ff8e7..0550be25f9d91 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceTests.java @@ -306,7 +306,16 @@ private static void bytesRefs(List suppliers) { })); suppliers.add(new TestCaseSupplier(List.of(DataType.GEO_SHAPE, DataType.INTEGER, DataType.INTEGER), () -> { - List field = randomList(1, 5, () -> new BytesRef(GEO.asWkt(GeometryTestUtils.randomGeometry(randomBoolean())))); + var pointCounter = new MvAppendTests.GeometryPointCountVisitor(); + List field = randomList( + 1, + 5, + () -> new BytesRef( + GEO.asWkt( + randomValueOtherThanMany(g -> g.visit(pointCounter) > 500, () -> GeometryTestUtils.randomGeometry(randomBoolean())) + ) + ) + ); int length = field.size(); int start = randomIntBetween(0, length - 1); int end = randomIntBetween(start, length - 1); @@ -323,7 +332,16 @@ private static void bytesRefs(List suppliers) { })); suppliers.add(new TestCaseSupplier(List.of(DataType.CARTESIAN_SHAPE, DataType.INTEGER, DataType.INTEGER), () -> { - List field = randomList(1, 5, () -> new BytesRef(CARTESIAN.asWkt(ShapeTestUtils.randomGeometry(randomBoolean())))); + var pointCounter = new MvAppendTests.GeometryPointCountVisitor(); + List field = randomList( + 1, + 5, + () -> new BytesRef( + CARTESIAN.asWkt( + randomValueOtherThanMany(g -> g.visit(pointCounter) > 500, () -> GeometryTestUtils.randomGeometry(randomBoolean())) + ) + ) + ); int length = field.size(); int start = randomIntBetween(0, length - 1); int end = randomIntBetween(start, length - 1); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortSerializationTests.java new file mode 100644 index 0000000000000..1728ad6f09357 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSortSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvSortSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvSort createTestInstance() { + Source source = randomSource(); + Expression field = randomChild(); + Expression order = randomBoolean() ? null : randomChild(); + return new MvSort(source, field, order); + } + + @Override + protected MvSort mutateInstance(MvSort instance) throws IOException { + Source source = instance.source(); + Expression field = instance.field(); + Expression order = instance.order(); + if (randomBoolean()) { + field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + } else { + order = randomValueOtherThan(order, () -> randomBoolean() ? null : randomChild()); + } + return new MvSort(source, field, order); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumSerializationTests.java new file mode 100644 index 0000000000000..e8ddcc9340b45 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSumSerializationTests.java @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvSumSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvSum createTestInstance() { + return new MvSum(randomSource(), randomChild()); + } + + @Override + protected MvSum mutateInstance(MvSum instance) throws IOException { + return new MvSum(instance.source(), randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild)); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipSerializationTests.java new file mode 100644 index 0000000000000..d16ca02627b29 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvZipSerializationTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.multivalue; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MvZipSerializationTests extends AbstractMvSerializationTests { + @Override + protected MvZip createTestInstance() { + Source source = randomSource(); + Expression mvLeft = randomChild(); + Expression mvRight = randomChild(); + Expression delim = randomBoolean() ? null : randomChild(); + return new MvZip(source, mvLeft, mvRight, delim); + } + + @Override + protected MvZip mutateInstance(MvZip instance) throws IOException { + Source source = instance.source(); + Expression mvLeft = instance.mvLeft(); + Expression mvRight = instance.mvRight(); + Expression delim = instance.delim(); + switch (between(0, 2)) { + case 0 -> mvLeft = randomValueOtherThan(mvLeft, AbstractExpressionSerializationTests::randomChild); + case 1 -> mvRight = randomValueOtherThan(mvRight, AbstractExpressionSerializationTests::randomChild); + case 2 -> delim = randomValueOtherThan(delim, () -> randomBoolean() ? null : randomChild()); + } + return new MvZip(source, mvLeft, mvRight, delim); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceSerializationTests.java new file mode 100644 index 0000000000000..7cab0a957b235 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/CoalesceSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class CoalesceSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Coalesce create(Source source, Expression first, List rest) { + return new Coalesce(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullSerializationTests.java new file mode 100644 index 0000000000000..23545b3627a1a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNotNullSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNotNull; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class IsNotNullSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return UnaryScalarFunction.getNamedWriteables(); + } + + @Override + protected IsNotNull createTestInstance() { + return new IsNotNull(randomSource(), randomChild()); + } + + @Override + protected IsNotNull mutateInstance(IsNotNull instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new IsNotNull(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullSerializationTests.java new file mode 100644 index 0000000000000..354a2129d7ec0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/nulls/IsNullSerializationTests.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.nulls; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.predicate.nulls.IsNull; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class IsNullSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return UnaryScalarFunction.getNamedWriteables(); + } + + @Override + protected IsNull createTestInstance() { + return new IsNull(randomSource(), randomChild()); + } + + @Override + protected IsNull mutateInstance(IsNull instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new IsNull(source, child); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatSerializationTests.java new file mode 100644 index 0000000000000..30f6acffbaf8a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ConcatSerializationTests.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractVarargsSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.util.List; + +public class ConcatSerializationTests extends AbstractVarargsSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected Concat create(Source source, Expression first, List rest) { + return new Concat(source, first, rest); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerSerializationTests.java new file mode 100644 index 0000000000000..f2dbdbd74470a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToLowerSerializationTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class ToLowerSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected ToLower createTestInstance() { + return new ToLower(randomSource(), randomChild(), configuration()); + } + + @Override + protected ToLower mutateInstance(ToLower instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new ToLower(source, child, configuration()); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperSerializationTests.java new file mode 100644 index 0000000000000..e57aedd79fdfd --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/string/ToUpperSerializationTests.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; +import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; + +import java.io.IOException; +import java.util.List; + +public class ToUpperSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected List getNamedWriteables() { + return EsqlScalarFunction.getNamedWriteables(); + } + + @Override + protected ToUpper createTestInstance() { + return new ToUpper(randomSource(), randomChild(), configuration()); + } + + @Override + protected ToUpper mutateInstance(ToUpper instance) throws IOException { + Source source = instance.source(); + Expression child = randomValueOtherThan(instance.field(), AbstractExpressionSerializationTests::randomChild); + return new ToUpper(source, child, configuration()); + } + + @Override + protected boolean alwaysEmptySource() { + return true; + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticSerializationTests.java new file mode 100644 index 0000000000000..c9a7933142605 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AbstractArithmeticSerializationTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; +import java.util.List; + +public abstract class AbstractArithmeticSerializationTests extends AbstractExpressionSerializationTests< + T> { + protected abstract T create(Source source, Expression left, Expression right); + + @Override + protected final T createTestInstance() { + return create(randomSource(), randomChild(), randomChild()); + } + + @Override + protected final T mutateInstance(T instance) throws IOException { + Expression left = instance.left(); + Expression right = instance.right(); + if (randomBoolean()) { + left = randomValueOtherThan(instance.left(), AbstractExpressionSerializationTests::randomChild); + } else { + right = randomValueOtherThan(instance.right(), AbstractExpressionSerializationTests::randomChild); + } + return create(instance.source(), left, right); + } + + @Override + protected List getNamedWriteables() { + return EsqlArithmeticOperation.getNamedWriteables(); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddSerializationTests.java new file mode 100644 index 0000000000000..b8924a01d0904 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/AddSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class AddSerializationTests extends AbstractArithmeticSerializationTests { + @Override + protected Add create(Source source, Expression left, Expression right) { + return new Add(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivSerializationTests.java new file mode 100644 index 0000000000000..b7e01eb835a47 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/DivSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class DivSerializationTests extends AbstractArithmeticSerializationTests
    { + @Override + protected Div create(Source source, Expression left, Expression right) { + return new Div(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModSerializationTests.java new file mode 100644 index 0000000000000..a0f072635db10 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/ModSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class ModSerializationTests extends AbstractArithmeticSerializationTests { + @Override + protected Mod create(Source source, Expression left, Expression right) { + return new Mod(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulSerializationTests.java new file mode 100644 index 0000000000000..d6eb1e3cf3cf0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/MulSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class MulSerializationTests extends AbstractArithmeticSerializationTests { + @Override + protected Mul create(Source source, Expression left, Expression right) { + return new Mul(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubSerializationTests.java new file mode 100644 index 0000000000000..274d7f669b2aa --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/arithmetic/SubSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class SubSerializationTests extends AbstractArithmeticSerializationTests { + @Override + protected Sub create(Source source, Expression left, Expression right) { + return new Sub(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/AbstractComparisonSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/AbstractComparisonSerializationTests.java new file mode 100644 index 0000000000000..8f28cfddb1d3a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/AbstractComparisonSerializationTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; +import java.util.List; + +public abstract class AbstractComparisonSerializationTests extends AbstractExpressionSerializationTests { + protected abstract T create(Source source, Expression left, Expression right); + + @Override + protected final T createTestInstance() { + return create(randomSource(), randomChild(), randomChild()); + } + + @Override + protected final T mutateInstance(T instance) throws IOException { + Expression left = instance.left(); + Expression right = instance.right(); + if (randomBoolean()) { + left = randomValueOtherThan(instance.left(), AbstractExpressionSerializationTests::randomChild); + } else { + right = randomValueOtherThan(instance.right(), AbstractExpressionSerializationTests::randomChild); + } + return create(instance.source(), left, right); + } + + @Override + protected List getNamedWriteables() { + return EsqlBinaryComparison.getNamedWriteables(); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSerializationTests.java new file mode 100644 index 0000000000000..cfeb8ce88ec87 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class EqualsSerializationTests extends AbstractComparisonSerializationTests { + @Override + protected Equals create(Source source, Expression left, Expression right) { + return new Equals(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualSerializationTests.java new file mode 100644 index 0000000000000..b8f9d510fb33a --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanOrEqualSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class GreaterThanOrEqualSerializationTests extends AbstractComparisonSerializationTests { + @Override + protected GreaterThanOrEqual create(Source source, Expression left, Expression right) { + return new GreaterThanOrEqual(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanSerializationTests.java new file mode 100644 index 0000000000000..93352f7f9d3e0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/GreaterThanSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class GreaterThanSerializationTests extends AbstractComparisonSerializationTests { + @Override + protected GreaterThan create(Source source, Expression left, Expression right) { + return new GreaterThan(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsSerializationTests.java new file mode 100644 index 0000000000000..d9daa27936267 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InsensitiveEqualsSerializationTests.java @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; +import java.util.List; + +public class InsensitiveEqualsSerializationTests extends AbstractExpressionSerializationTests { + @Override + protected final InsensitiveEquals createTestInstance() { + return new InsensitiveEquals(randomSource(), randomChild(), randomChild()); + } + + @Override + protected final InsensitiveEquals mutateInstance(InsensitiveEquals instance) throws IOException { + Expression left = instance.left(); + Expression right = instance.right(); + if (randomBoolean()) { + left = randomValueOtherThan(instance.left(), AbstractExpressionSerializationTests::randomChild); + } else { + right = randomValueOtherThan(instance.right(), AbstractExpressionSerializationTests::randomChild); + } + return new InsensitiveEquals(instance.source(), left, right); + } + + @Override + protected List getNamedWriteables() { + return List.of(InsensitiveEquals.ENTRY); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualSerializationTests.java new file mode 100644 index 0000000000000..f7580a23bf47d --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanOrEqualSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class LessThanOrEqualSerializationTests extends AbstractComparisonSerializationTests { + @Override + protected LessThanOrEqual create(Source source, Expression left, Expression right) { + return new LessThanOrEqual(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanSerializationTests.java new file mode 100644 index 0000000000000..220f56ebb7c00 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class LessThanSerializationTests extends AbstractComparisonSerializationTests { + @Override + protected LessThan create(Source source, Expression left, Expression right) { + return new LessThan(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsSerializationTests.java new file mode 100644 index 0000000000000..be6c880e08736 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsSerializationTests.java @@ -0,0 +1,18 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison; + +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.tree.Source; + +public class NotEqualsSerializationTests extends AbstractComparisonSerializationTests { + @Override + protected NotEquals create(Source source, Expression left, Expression right) { + return new NotEquals(source, left, right); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/AbstractFulltextSerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/AbstractFulltextSerializationTests.java new file mode 100644 index 0000000000000..88f88436f8a04 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/AbstractFulltextSerializationTests.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.fulltext; + +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.FullTextPredicate; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public abstract class AbstractFulltextSerializationTests extends AbstractExpressionSerializationTests { + + static final String OPTION_DELIMITER = ";"; + + @Override + protected List getNamedWriteables() { + return FullTextPredicate.getNamedWriteables(); + } + + String randomOptionOrNull() { + if (randomBoolean()) { + return null; + } + HashMap options = new HashMap<>(); + int maxOptions = randomInt(8); + for (int i = 0; i < maxOptions; i++) { + var opt = randomIndividualOption(); + options.computeIfAbsent(opt.getKey(), k -> opt.getValue()); // no duplicate options + } + return options.entrySet().stream().map(e -> e.getKey() + "=" + e.getValue()).collect(Collectors.joining(OPTION_DELIMITER)); + } + + Map.Entry randomIndividualOption() { + return Map.entry(randomAlphaOfLength(randomIntBetween(1, 4)), randomAlphaOfLength(randomIntBetween(1, 4))); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MatchQuerySerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MatchQuerySerializationTests.java new file mode 100644 index 0000000000000..80a538cf84baa --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MatchQuerySerializationTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.fulltext; + +import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate; +import org.elasticsearch.xpack.esql.expression.AbstractExpressionSerializationTests; + +import java.io.IOException; + +public class MatchQuerySerializationTests extends AbstractFulltextSerializationTests { + + @Override + protected final MatchQueryPredicate createTestInstance() { + return new MatchQueryPredicate(randomSource(), randomChild(), randomAlphaOfLength(randomIntBetween(1, 16)), randomOptionOrNull()); + } + + @Override + protected MatchQueryPredicate mutateInstance(MatchQueryPredicate instance) throws IOException { + var field = instance.field(); + var query = instance.query(); + var options = instance.options(); + switch (between(0, 2)) { + case 0 -> field = randomValueOtherThan(field, AbstractExpressionSerializationTests::randomChild); + case 1 -> query = randomValueOtherThan(query, () -> randomAlphaOfLength(randomIntBetween(1, 16))); + case 2 -> options = randomValueOtherThan(options, this::randomOptionOrNull); + } + return new MatchQueryPredicate(instance.source(), field, query, options); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MultiMatchQuerySerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MultiMatchQuerySerializationTests.java new file mode 100644 index 0000000000000..d4d0f2edc11b1 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/MultiMatchQuerySerializationTests.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.fulltext; + +import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Collectors; + +public class MultiMatchQuerySerializationTests extends AbstractFulltextSerializationTests { + + @Override + protected final MultiMatchQueryPredicate createTestInstance() { + return new MultiMatchQueryPredicate( + randomSource(), + randomFieldString(), + randomAlphaOfLength(randomIntBetween(1, 16)), + randomOptionOrNull() + ); + } + + @Override + protected MultiMatchQueryPredicate mutateInstance(MultiMatchQueryPredicate instance) throws IOException { + var fieldString = instance.fieldString(); + var query = instance.query(); + var options = instance.options(); + switch (between(0, 2)) { + case 0 -> fieldString = randomValueOtherThan(fieldString, this::randomFieldString); + case 1 -> query = randomValueOtherThan(query, () -> randomAlphaOfLength(randomIntBetween(1, 16))); + case 2 -> options = randomValueOtherThan(options, this::randomOptionOrNull); + } + return new MultiMatchQueryPredicate(instance.source(), fieldString, query, options); + } + + String randomFieldString() { + if (randomBoolean()) { + return ""; // empty, no fields + } + HashMap fields = new HashMap<>(); + int maxOptions = randomInt(4); + for (int i = 0; i < maxOptions; i++) { + var opt = randomIndividualField(); + fields.computeIfAbsent(opt.getKey(), k -> opt.getValue()); // no duplicate fields + } + return fields.entrySet().stream().map(e -> e.getKey() + "^" + e.getValue()).collect(Collectors.joining(",")); + } + + Map.Entry randomIndividualField() { + return Map.entry(randomAlphaOfLength(randomIntBetween(1, 4)), randomFloat()); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/StringQuerySerializationTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/StringQuerySerializationTests.java new file mode 100644 index 0000000000000..ff00a161e1bb1 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/fulltext/StringQuerySerializationTests.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.expression.predicate.operator.fulltext; + +import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.StringQueryPredicate; + +import java.io.IOException; + +public class StringQuerySerializationTests extends AbstractFulltextSerializationTests { + + private static final String COMMA = ","; + + @Override + protected final StringQueryPredicate createTestInstance() { + return new StringQueryPredicate(randomSource(), randomAlphaOfLength(randomIntBetween(1, 16)), randomOptionOrNull()); + } + + @Override + protected StringQueryPredicate mutateInstance(StringQueryPredicate instance) throws IOException { + var query = instance.query(); + var options = instance.options(); + if (randomBoolean()) { + query = randomValueOtherThan(query, () -> randomAlphaOfLength(randomIntBetween(1, 16))); + } else { + options = randomValueOtherThan(options, this::randomOptionOrNull); + } + return new StringQueryPredicate(instance.source(), query, options); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java index 2278be659c538..57d304a4f032e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/io/stream/PlanNamedTypesTests.java @@ -19,11 +19,11 @@ import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.SerializationTestUtils; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NameId; import org.elasticsearch.xpack.esql.core.expression.Nullability; -import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.arithmetic.ArithmeticOperation; import org.elasticsearch.xpack.esql.core.index.EsIndex; import org.elasticsearch.xpack.esql.core.plan.logical.Filter; @@ -110,6 +110,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Stream; import static org.elasticsearch.test.ListMatcher.matchesList; @@ -194,12 +195,12 @@ public void testLogicalPlanEntries() { public void testFunctionEntries() { var serializableFunctions = PlanNamedTypes.namedTypeEntries() .stream() - .filter(e -> Function.class.isAssignableFrom(e.categoryClass())) + .filter(e -> Expression.class.isAssignableFrom(e.categoryClass())) .map(PlanNameRegistry.Entry::name) .sorted() .toList(); - for (var function : (new EsqlFunctionRegistry()).listFunctions()) { + for (var function : new EsqlFunctionRegistry().listFunctions()) { assertThat(serializableFunctions, hasItem(equalTo(PlanNamedTypes.name(function.clazz())))); } } @@ -233,15 +234,13 @@ public void testBinComparisonSimple() throws IOException { var orig = new Equals(Source.EMPTY, field("foo", DataType.DOUBLE), field("bar", DataType.DOUBLE)); BytesStreamOutput bso = new BytesStreamOutput(); PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null); - out.writeNamed(EsqlBinaryComparison.class, orig); - var deser = (Equals) planStreamInput(bso).readNamed(EsqlBinaryComparison.class); + out.writeNamed(Expression.class, orig); + var deser = (Equals) planStreamInput(bso).readNamed(Expression.class); EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); } public void testBinComparison() { - Stream.generate(PlanNamedTypesTests::randomBinaryComparison) - .limit(100) - .forEach(obj -> assertNamedType(EsqlBinaryComparison.class, obj)); + Stream.generate(PlanNamedTypesTests::randomBinaryComparison).limit(100).forEach(obj -> assertNamedType(Expression.class, obj)); } public void testAggFunctionSimple() throws IOException { @@ -261,15 +260,13 @@ public void testArithmeticOperationSimple() throws IOException { var orig = new Add(Source.EMPTY, field("foo", DataType.LONG), field("bar", DataType.LONG)); BytesStreamOutput bso = new BytesStreamOutput(); PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null); - out.writeNamed(ArithmeticOperation.class, orig); - var deser = (Add) planStreamInput(bso).readNamed(ArithmeticOperation.class); + out.writeNamed(Expression.class, orig); + var deser = (Add) planStreamInput(bso).readNamed(Expression.class); EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); } public void testArithmeticOperation() { - Stream.generate(PlanNamedTypesTests::randomArithmeticOperation) - .limit(100) - .forEach(obj -> assertNamedType(ArithmeticOperation.class, obj)); + Stream.generate(PlanNamedTypesTests::randomArithmeticOperation).limit(100).forEach(obj -> assertNamedType(Expression.class, obj)); } public void testSubStringSimple() throws IOException { @@ -308,24 +305,6 @@ public void testPowSimple() throws IOException { EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); } - public void testLiteralSimple() throws IOException { - var orig = new Literal(Source.EMPTY, 1, DataType.INTEGER); - BytesStreamOutput bso = new BytesStreamOutput(); - PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null); - PlanNamedTypes.writeLiteral(out, orig); - var deser = PlanNamedTypes.readLiteral(planStreamInput(bso)); - EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); - } - - public void testOrderSimple() throws IOException { - var orig = new Order(Source.EMPTY, field("val", DataType.INTEGER), Order.OrderDirection.ASC, Order.NullsPosition.FIRST); - BytesStreamOutput bso = new BytesStreamOutput(); - PlanStreamOutput out = new PlanStreamOutput(bso, planNameRegistry, null); - PlanNamedTypes.writeOrder(out, orig); - var deser = (Order) PlanNamedTypes.readOrder(planStreamInput(bso)); - EqualsHashCodeTestUtils.checkEqualsAndHashCode(orig, unused -> deser); - } - public void testFieldSortSimple() throws IOException { var orig = new EsQueryExec.FieldSort(field("val", DataType.LONG), Order.OrderDirection.ASC, Order.NullsPosition.FIRST); BytesStreamOutput bso = new BytesStreamOutput(); @@ -404,10 +383,14 @@ private static void assertNamedType(Class type, T origObj) { } static EsIndex randomEsIndex() { + Set concreteIndices = new TreeSet<>(); + while (concreteIndices.size() < 2) { + concreteIndices.add(randomAlphaOfLengthBetween(1, 25)); + } return new EsIndex( - randomAlphaOfLength(randomIntBetween(1, 25)), - Map.of(randomAlphaOfLength(randomIntBetween(1, 25)), randomKeywordEsField()), - Set.of(randomAlphaOfLength(randomIntBetween(1, 25)), randomAlphaOfLength(randomIntBetween(1, 25))) + randomAlphaOfLengthBetween(1, 25), + Map.of(randomAlphaOfLengthBetween(1, 25), randomKeywordEsField()), + concreteIndices ); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/FoldNull.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/FoldNull.java index 4e15cd27a50fa..dc12f0231b79c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/FoldNull.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/FoldNull.java @@ -9,7 +9,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; -public class FoldNull extends LogicalPlanOptimizer.FoldNull { +public class FoldNull extends org.elasticsearch.xpack.esql.optimizer.rules.FoldNull { @Override public Expression rule(Expression e) { return super.rule(e); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 8e2ed96607e0d..74bdcf824ba80 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -106,6 +106,9 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.optimizer.rules.LiteralsOnTheRight; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineFilters; +import org.elasticsearch.xpack.esql.optimizer.rules.PushDownAndCombineLimits; +import org.elasticsearch.xpack.esql.optimizer.rules.SplitInWithFoldableValue; import org.elasticsearch.xpack.esql.parser.EsqlParser; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; @@ -704,7 +707,7 @@ public void testCombineLimits() { var anotherLimit = new Limit(EMPTY, L(limitValues[secondLimit]), oneLimit); assertEquals( new Limit(EMPTY, L(Math.min(limitValues[0], limitValues[1])), emptySource()), - new LogicalPlanOptimizer.PushDownAndCombineLimits().rule(anotherLimit) + new PushDownAndCombineLimits().rule(anotherLimit) ); } @@ -747,10 +750,7 @@ public void testCombineFilters() { Filter fa = new Filter(EMPTY, relation, conditionA); Filter fb = new Filter(EMPTY, fa, conditionB); - assertEquals( - new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB)), - new LogicalPlanOptimizer.PushDownAndCombineFilters().apply(fb) - ); + assertEquals(new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB)), new PushDownAndCombineFilters().apply(fb)); } public void testCombineFiltersLikeRLike() { @@ -761,10 +761,7 @@ public void testCombineFiltersLikeRLike() { Filter fa = new Filter(EMPTY, relation, conditionA); Filter fb = new Filter(EMPTY, fa, conditionB); - assertEquals( - new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB)), - new LogicalPlanOptimizer.PushDownAndCombineFilters().apply(fb) - ); + assertEquals(new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB)), new PushDownAndCombineFilters().apply(fb)); } public void testPushDownFilter() { @@ -778,7 +775,7 @@ public void testPushDownFilter() { Filter fb = new Filter(EMPTY, keep, conditionB); Filter combinedFilter = new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB)); - assertEquals(new EsqlProject(EMPTY, combinedFilter, projections), new LogicalPlanOptimizer.PushDownAndCombineFilters().apply(fb)); + assertEquals(new EsqlProject(EMPTY, combinedFilter, projections), new PushDownAndCombineFilters().apply(fb)); } public void testPushDownLikeRlikeFilter() { @@ -792,7 +789,7 @@ public void testPushDownLikeRlikeFilter() { Filter fb = new Filter(EMPTY, keep, conditionB); Filter combinedFilter = new Filter(EMPTY, relation, new And(EMPTY, conditionA, conditionB)); - assertEquals(new EsqlProject(EMPTY, combinedFilter, projections), new LogicalPlanOptimizer.PushDownAndCombineFilters().apply(fb)); + assertEquals(new EsqlProject(EMPTY, combinedFilter, projections), new PushDownAndCombineFilters().apply(fb)); } // from ... | where a > 1 | stats count(1) by b | where count(1) >= 3 and b < 2 @@ -819,7 +816,7 @@ public void testSelectivelyPushDownFilterPastFunctionAgg() { ), aggregateCondition ); - assertEquals(expected, new LogicalPlanOptimizer.PushDownAndCombineFilters().apply(fb)); + assertEquals(expected, new PushDownAndCombineFilters().apply(fb)); } public void testSelectivelyPushDownFilterPastRefAgg() { @@ -2214,7 +2211,7 @@ public void testSplittingInWithFoldableValue() { FieldAttribute fa = getFieldAttribute("foo"); In in = new In(EMPTY, ONE, List.of(TWO, THREE, fa, L(null))); Or expected = new Or(EMPTY, new In(EMPTY, ONE, List.of(TWO, THREE)), new In(EMPTY, ONE, List.of(fa, L(null)))); - assertThat(new LogicalPlanOptimizer.SplitInWithFoldableValue().rule(in), equalTo(expected)); + assertThat(new SplitInWithFoldableValue().rule(in), equalTo(expected)); } public void testReplaceFilterWithExact() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java index ff183f2e6b37f..b550f6e6090da 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/OptimizerRulesTests.java @@ -48,12 +48,12 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; -import org.elasticsearch.xpack.esql.optimizer.LogicalPlanOptimizer.ReplaceRegexMatch; import org.elasticsearch.xpack.esql.optimizer.rules.BooleanFunctionEqualsElimination; import org.elasticsearch.xpack.esql.optimizer.rules.CombineDisjunctionsToIn; import org.elasticsearch.xpack.esql.optimizer.rules.ConstantFolding; import org.elasticsearch.xpack.esql.optimizer.rules.LiteralsOnTheRight; import org.elasticsearch.xpack.esql.optimizer.rules.PropagateEquals; +import org.elasticsearch.xpack.esql.optimizer.rules.ReplaceRegexMatch; import java.util.List; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PropagateNullable.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PropagateNullable.java index bd2bb91cd2ea4..a7a996230facd 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PropagateNullable.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/PropagateNullable.java @@ -10,7 +10,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And; -public class PropagateNullable extends LogicalPlanOptimizer.PropagateNullable { +public class PropagateNullable extends org.elasticsearch.xpack.esql.optimizer.rules.PropagateNullable { @Override public Expression rule(And and) { return super.rule(and); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java new file mode 100644 index 0000000000000..97fb145d4c2e4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/AbstractStatementParserTests.java @@ -0,0 +1,132 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.parser; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; +import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.core.type.DataType; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; +import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +abstract class AbstractStatementParserTests extends ESTestCase { + + EsqlParser parser = new EsqlParser(); + + void assertStatement(String statement, LogicalPlan expected) { + final LogicalPlan actual; + try { + actual = statement(statement); + } catch (Exception e) { + throw new AssertionError("parsing error for [" + statement + "]", e); + } + assertThat(statement, actual, equalTo(expected)); + } + + LogicalPlan statement(String e) { + return statement(e, QueryParams.EMPTY); + } + + LogicalPlan statement(String e, QueryParams params) { + return parser.createStatement(e, params); + } + + LogicalPlan processingCommand(String e) { + return parser.createStatement("row a = 1 | " + e); + } + + static UnresolvedAttribute attribute(String name) { + return new UnresolvedAttribute(EMPTY, name); + } + + static ReferenceAttribute referenceAttribute(String name, DataType type) { + return new ReferenceAttribute(EMPTY, name, type); + } + + static Literal integer(int i) { + return new Literal(EMPTY, i, DataType.INTEGER); + } + + static Literal integers(int... ints) { + return new Literal(EMPTY, Arrays.stream(ints).boxed().toList(), DataType.INTEGER); + } + + static Literal literalLong(long i) { + return new Literal(EMPTY, i, DataType.LONG); + } + + static Literal literalLongs(long... longs) { + return new Literal(EMPTY, Arrays.stream(longs).boxed().toList(), DataType.LONG); + } + + static Literal literalDouble(double d) { + return new Literal(EMPTY, d, DataType.DOUBLE); + } + + static Literal literalDoubles(double... doubles) { + return new Literal(EMPTY, Arrays.stream(doubles).boxed().toList(), DataType.DOUBLE); + } + + static Literal literalUnsignedLong(String ulong) { + return new Literal(EMPTY, asLongUnsigned(new BigInteger(ulong)), DataType.UNSIGNED_LONG); + } + + static Literal literalUnsignedLongs(String... ulongs) { + return new Literal(EMPTY, Arrays.stream(ulongs).map(s -> asLongUnsigned(new BigInteger(s))).toList(), DataType.UNSIGNED_LONG); + } + + static Literal literalBoolean(boolean b) { + return new Literal(EMPTY, b, DataType.BOOLEAN); + } + + static Literal literalBooleans(boolean... booleans) { + List v = new ArrayList<>(booleans.length); + for (boolean b : booleans) { + v.add(b); + } + return new Literal(EMPTY, v, DataType.BOOLEAN); + } + + static Literal literalString(String s) { + return new Literal(EMPTY, s, DataType.KEYWORD); + } + + static Literal literalStrings(String... strings) { + return new Literal(EMPTY, Arrays.asList(strings), DataType.KEYWORD); + } + + void expectError(String query, String errorMessage) { + ParsingException e = expectThrows(ParsingException.class, "Expected syntax error for " + query, () -> statement(query)); + assertThat(e.getMessage(), containsString(errorMessage)); + } + + void expectVerificationError(String query, String errorMessage) { + VerificationException e = expectThrows(VerificationException.class, "Expected syntax error for " + query, () -> statement(query)); + assertThat(e.getMessage(), containsString(errorMessage)); + } + + void expectError(String query, List params, String errorMessage) { + ParsingException e = expectThrows( + ParsingException.class, + "Expected syntax error for " + query, + () -> statement(query, new QueryParams(params)) + ); + assertThat(e.getMessage(), containsString(errorMessage)); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java index 28662c2470f15..5251d7ed03d81 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/StatementParserTests.java @@ -10,8 +10,6 @@ import org.elasticsearch.Build; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.core.capabilities.UnresolvedException; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.EmptyAttribute; @@ -19,7 +17,6 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.Order; -import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.logical.Not; @@ -29,7 +26,6 @@ import org.elasticsearch.xpack.esql.core.plan.logical.Limit; import org.elasticsearch.xpack.esql.core.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.core.plan.logical.OrderBy; -import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike; import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; @@ -51,9 +47,6 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Row; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -65,7 +58,6 @@ import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; -import static org.elasticsearch.xpack.esql.core.util.NumericUtils.asLongUnsigned; import static org.elasticsearch.xpack.esql.parser.ExpressionBuilder.breakIntoFragments; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; @@ -76,10 +68,9 @@ import static org.hamcrest.Matchers.is; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") -public class StatementParserTests extends ESTestCase { +public class StatementParserTests extends AbstractStatementParserTests { private static String FROM = "from test"; - EsqlParser parser = new EsqlParser(); public void testRowCommand() { assertEquals( @@ -1388,106 +1379,6 @@ public void testMetricWithGroupKeyAsAgg() { } } - private void assertStatement(String statement, LogicalPlan expected) { - final LogicalPlan actual; - try { - actual = statement(statement); - } catch (Exception e) { - throw new AssertionError("parsing error for [" + statement + "]", e); - } - assertThat(statement, actual, equalTo(expected)); - } - - private LogicalPlan statement(String e) { - return statement(e, QueryParams.EMPTY); - } - - private LogicalPlan statement(String e, QueryParams params) { - return parser.createStatement(e, params); - } - - private LogicalPlan processingCommand(String e) { - return parser.createStatement("row a = 1 | " + e); - } - private static final LogicalPlan PROCESSING_CMD_INPUT = new Row(EMPTY, List.of(new Alias(EMPTY, "a", integer(1)))); - private static UnresolvedAttribute attribute(String name) { - return new UnresolvedAttribute(EMPTY, name); - } - - private static ReferenceAttribute referenceAttribute(String name, DataType type) { - return new ReferenceAttribute(EMPTY, name, type); - } - - private static Literal integer(int i) { - return new Literal(EMPTY, i, DataType.INTEGER); - } - - private static Literal integers(int... ints) { - return new Literal(EMPTY, Arrays.stream(ints).boxed().toList(), DataType.INTEGER); - } - - private static Literal literalLong(long i) { - return new Literal(EMPTY, i, DataType.LONG); - } - - private static Literal literalLongs(long... longs) { - return new Literal(EMPTY, Arrays.stream(longs).boxed().toList(), DataType.LONG); - } - - private static Literal literalDouble(double d) { - return new Literal(EMPTY, d, DataType.DOUBLE); - } - - private static Literal literalDoubles(double... doubles) { - return new Literal(EMPTY, Arrays.stream(doubles).boxed().toList(), DataType.DOUBLE); - } - - private static Literal literalUnsignedLong(String ulong) { - return new Literal(EMPTY, asLongUnsigned(new BigInteger(ulong)), DataType.UNSIGNED_LONG); - } - - private static Literal literalUnsignedLongs(String... ulongs) { - return new Literal(EMPTY, Arrays.stream(ulongs).map(s -> asLongUnsigned(new BigInteger(s))).toList(), DataType.UNSIGNED_LONG); - } - - private static Literal literalBoolean(boolean b) { - return new Literal(EMPTY, b, DataType.BOOLEAN); - } - - private static Literal literalBooleans(boolean... booleans) { - List v = new ArrayList<>(booleans.length); - for (boolean b : booleans) { - v.add(b); - } - return new Literal(EMPTY, v, DataType.BOOLEAN); - } - - private static Literal literalString(String s) { - return new Literal(EMPTY, s, DataType.KEYWORD); - } - - private static Literal literalStrings(String... strings) { - return new Literal(EMPTY, Arrays.asList(strings), DataType.KEYWORD); - } - - private void expectError(String query, String errorMessage) { - ParsingException e = expectThrows(ParsingException.class, "Expected syntax error for " + query, () -> statement(query)); - assertThat(e.getMessage(), containsString(errorMessage)); - } - - private void expectVerificationError(String query, String errorMessage) { - VerificationException e = expectThrows(VerificationException.class, "Expected syntax error for " + query, () -> statement(query)); - assertThat(e.getMessage(), containsString(errorMessage)); - } - - private void expectError(String query, List params, String errorMessage) { - ParsingException e = expectThrows( - ParsingException.class, - "Expected syntax error for " + query, - () -> statement(query, new QueryParams(params)) - ); - assertThat(e.getMessage(), containsString(errorMessage)); - } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java index 8d1353cbddd42..17dca8096de0f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/session/IndexResolverFieldNamesTests.java @@ -13,8 +13,8 @@ import java.util.Collections; import java.util.Set; -import static org.elasticsearch.xpack.esql.core.index.IndexResolver.ALL_FIELDS; -import static org.elasticsearch.xpack.esql.core.index.IndexResolver.INDEX_METADATA_FIELD; +import static org.elasticsearch.xpack.esql.session.IndexResolver.ALL_FIELDS; +import static org.elasticsearch.xpack.esql.session.IndexResolver.INDEX_METADATA_FIELD; import static org.hamcrest.Matchers.equalTo; public class IndexResolverFieldNamesTests extends ESTestCase { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java index d3011506bb5ef..5883d41f32125 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/PlanExecutorMetricsTests.java @@ -20,11 +20,10 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlQueryRequest; import org.elasticsearch.xpack.esql.analysis.EnrichResolution; -import org.elasticsearch.xpack.esql.core.index.IndexResolver; import org.elasticsearch.xpack.esql.enrich.EnrichPolicyResolver; import org.elasticsearch.xpack.esql.execution.PlanExecutor; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; -import org.elasticsearch.xpack.esql.session.EsqlIndexResolver; +import org.elasticsearch.xpack.esql.session.IndexResolver; import org.elasticsearch.xpack.esql.type.EsqlDataTypeRegistry; import org.junit.After; import org.junit.Before; @@ -34,7 +33,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.hamcrest.Matchers.instanceOf; @@ -73,7 +71,7 @@ public void testFailedMetric() { String[] indices = new String[] { "test" }; Client qlClient = mock(Client.class); - IndexResolver idxResolver = new IndexResolver(qlClient, randomAlphaOfLength(10), EsqlDataTypeRegistry.INSTANCE, Set::of); + IndexResolver idxResolver = new IndexResolver(qlClient, EsqlDataTypeRegistry.INSTANCE); // simulate a valid field_caps response so we can parse and correctly analyze de query FieldCapabilitiesResponse fieldCapabilitiesResponse = mock(FieldCapabilitiesResponse.class); when(fieldCapabilitiesResponse.getIndices()).thenReturn(indices); @@ -87,7 +85,7 @@ public void testFailedMetric() { }).when(qlClient).fieldCaps(any(), any()); Client esqlClient = mock(Client.class); - EsqlIndexResolver esqlIndexResolver = new EsqlIndexResolver(esqlClient, EsqlDataTypeRegistry.INSTANCE); + IndexResolver indexResolver = new IndexResolver(esqlClient, EsqlDataTypeRegistry.INSTANCE); doAnswer((Answer) invocation -> { @SuppressWarnings("unchecked") ActionListener listener = (ActionListener) invocation.getArguments()[1]; @@ -96,7 +94,7 @@ public void testFailedMetric() { return null; }).when(esqlClient).fieldCaps(any(), any()); - var planExecutor = new PlanExecutor(idxResolver, esqlIndexResolver); + var planExecutor = new PlanExecutor(indexResolver); var enrichResolver = mockEnrichResolver(); var request = new EsqlQueryRequest(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java index 7dca73219d6a1..ad7be1e38681f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeRegistryTests.java @@ -14,7 +14,7 @@ import org.elasticsearch.xpack.esql.core.index.IndexResolution; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; -import org.elasticsearch.xpack.esql.session.EsqlIndexResolver; +import org.elasticsearch.xpack.esql.session.IndexResolver; import java.util.List; import java.util.Map; @@ -51,7 +51,7 @@ private void resolve(String esTypeName, TimeSeriesParams.MetricType metricType, ); FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse(idxResponses, List.of()); - IndexResolution resolution = new EsqlIndexResolver(null, EsqlDataTypeRegistry.INSTANCE).mergedMappings("idx-*", caps); + IndexResolution resolution = new IndexResolver(null, EsqlDataTypeRegistry.INSTANCE).mergedMappings("idx-*", caps); EsField f = resolution.get().mapping().get(field); assertThat(f.getDataType(), equalTo(expected)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java new file mode 100644 index 0000000000000..86baee58ca53f --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java @@ -0,0 +1,188 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.type; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.test.AbstractNamedWriteableTestCase; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.Expression; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToBoolean; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianPoint; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToCartesianShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToGeoPoint; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToGeoShape; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToIP; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToString; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToVersion; +import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; +import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; +import org.elasticsearch.xpack.esql.session.EsqlConfiguration; +import org.elasticsearch.xpack.esql.session.EsqlConfigurationSerializationTests; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.xpack.esql.type.EsqlDataTypes.isString; + +/** + * This test was originally based on the tests for sub-classes of EsField, like InvalidMappedFieldTests. + * However, it has a few important differences: + *
      + *
    • It is not in the esql.core module, but in the esql module, in order to have access to the sub-classes of AbstractConvertFunction, + * like ToString, which are important conversion Expressions used in the union-types feature.
      • + *
    • It extends AbstractNamedWriteableTestCase instead of AbstractEsFieldTypeTests, + * in order to wrap the StreamInput with a PlanStreamInput, since Expression is not yet fully supported in the new + * serialization approach (NamedWritable).
      • + *
    + * These differences can be minimized once Expression is fully supported in the new serialization approach, and the esql and esql.core + * modules are merged, or at least the relevant classes are moved. + */ +public class MultiTypeEsFieldTests extends AbstractNamedWriteableTestCase { + + private EsqlConfiguration config; + + @Before + public void initConfig() { + config = EsqlConfigurationSerializationTests.randomConfiguration(); + } + + @Override + protected MultiTypeEsField createTestInstance() { + String name = randomAlphaOfLength(4); + boolean toString = randomBoolean(); + DataType dataType = randomFrom(types()); + DataType toType = toString ? DataType.KEYWORD : dataType; + Map indexToConvertExpressions = randomConvertExpressions(name, toString, dataType); + return new MultiTypeEsField(name, toType, false, indexToConvertExpressions); + } + + @Override + protected MultiTypeEsField mutateInstance(MultiTypeEsField instance) throws IOException { + String name = instance.getName(); + DataType dataType = instance.getDataType(); + Map indexToConvertExpressions = instance.getIndexToConversionExpressions(); + switch (between(0, 2)) { + case 0 -> name = randomAlphaOfLength(name.length() + 1); + case 1 -> dataType = randomValueOtherThan(dataType, () -> randomFrom(DataType.types())); + case 2 -> indexToConvertExpressions = mutateConvertExpressions(name, dataType, indexToConvertExpressions); + default -> throw new IllegalArgumentException(); + } + return new MultiTypeEsField(name, dataType, false, indexToConvertExpressions); + } + + @Override + protected final NamedWriteableRegistry getNamedWriteableRegistry() { + List entries = new ArrayList<>(UnaryScalarFunction.getNamedWriteables()); + entries.addAll(Attribute.getNamedWriteables()); + entries.addAll(EsField.getNamedWriteables()); + entries.add(new NamedWriteableRegistry.Entry(MultiTypeEsField.class, "MultiTypeEsField", MultiTypeEsField::new)); + return new NamedWriteableRegistry(entries); + } + + @Override + protected final Class categoryClass() { + return MultiTypeEsField.class; + } + + @Override + protected final MultiTypeEsField copyInstance(MultiTypeEsField instance, TransportVersion version) throws IOException { + return copyInstance( + instance, + getNamedWriteableRegistry(), + (out, v) -> new PlanStreamOutput(out, new PlanNameRegistry(), config).writeNamedWriteable(v), + in -> { + PlanStreamInput pin = new PlanStreamInput(in, new PlanNameRegistry(), in.namedWriteableRegistry(), config); + return pin.readNamedWriteable(MultiTypeEsField.class); + }, + version + ); + } + + private static Map randomConvertExpressions(String name, boolean toString, DataType dataType) { + Map indexToConvertExpressions = new HashMap<>(); + if (toString) { + indexToConvertExpressions.put(randomAlphaOfLength(4), new ToString(Source.EMPTY, fieldAttribute(name, dataType))); + indexToConvertExpressions.put(randomAlphaOfLength(4), new ToString(Source.EMPTY, fieldAttribute(name, DataType.KEYWORD))); + } else { + indexToConvertExpressions.put(randomAlphaOfLength(4), testConvertExpression(name, DataType.KEYWORD, dataType)); + indexToConvertExpressions.put(randomAlphaOfLength(4), testConvertExpression(name, dataType, dataType)); + } + return indexToConvertExpressions; + } + + private Map mutateConvertExpressions( + String name, + DataType toType, + Map indexToConvertExpressions + ) { + return randomValueOtherThan( + indexToConvertExpressions, + () -> randomConvertExpressions(name, toType == DataType.KEYWORD, randomFrom(types())) + ); + } + + private static List types() { + return List.of( + DataType.BOOLEAN, + DataType.DATETIME, + DataType.DOUBLE, + DataType.FLOAT, + DataType.INTEGER, + DataType.IP, + DataType.KEYWORD, + DataType.LONG, + DataType.GEO_POINT, + DataType.GEO_SHAPE, + DataType.CARTESIAN_POINT, + DataType.CARTESIAN_SHAPE, + DataType.VERSION + ); + } + + private static Expression testConvertExpression(String name, DataType fromType, DataType toType) { + FieldAttribute fromField = fieldAttribute(name, fromType); + if (isString(toType)) { + return new ToString(Source.EMPTY, fromField); + } else { + return switch (toType) { + case BOOLEAN -> new ToBoolean(Source.EMPTY, fromField); + case DATETIME -> new ToDatetime(Source.EMPTY, fromField); + case DOUBLE, FLOAT -> new ToDouble(Source.EMPTY, fromField); + case INTEGER -> new ToInteger(Source.EMPTY, fromField); + case LONG -> new ToLong(Source.EMPTY, fromField); + case IP -> new ToIP(Source.EMPTY, fromField); + case KEYWORD -> new ToString(Source.EMPTY, fromField); + case GEO_POINT -> new ToGeoPoint(Source.EMPTY, fromField); + case GEO_SHAPE -> new ToGeoShape(Source.EMPTY, fromField); + case CARTESIAN_POINT -> new ToCartesianPoint(Source.EMPTY, fromField); + case CARTESIAN_SHAPE -> new ToCartesianShape(Source.EMPTY, fromField); + case VERSION -> new ToVersion(Source.EMPTY, fromField); + default -> throw new UnsupportedOperationException("Conversion from " + fromType + " to " + toType + " is not supported"); + }; + } + } + + private static FieldAttribute fieldAttribute(String name, DataType dataType) { + return new FieldAttribute(Source.EMPTY, name, new EsField(name, dataType, Map.of(), true)); + } +} diff --git a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java index 7a420aa41ce76..d4ecff4238591 100644 --- a/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java +++ b/x-pack/plugin/ilm/qa/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/ilm/actions/DownsampleActionIT.java @@ -18,12 +18,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.test.junit.annotations.TestLogging; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -54,7 +54,6 @@ import static org.elasticsearch.xpack.TimeSeriesRestDriver.getStepKeyForIndex; import static org.elasticsearch.xpack.TimeSeriesRestDriver.index; import static org.elasticsearch.xpack.TimeSeriesRestDriver.rolloverMaxOneDocCondition; -import static org.elasticsearch.xpack.TimeSeriesRestDriver.updatePolicy; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -158,11 +157,13 @@ public void updatePollInterval() throws IOException { updateClusterSettings(client(), Settings.builder().put("indices.lifecycle.poll_interval", "5s").build()); } - private void createIndex(String index, String alias, boolean isTimeSeries) throws IOException { + private void createIndex(String index, String alias, @Nullable String policy, boolean isTimeSeries) throws IOException { Settings.Builder settings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .put(LifecycleSettings.LIFECYCLE_NAME, policy); + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); + if (policy != null) { + settings.put(LifecycleSettings.LIFECYCLE_NAME, policy); + } if (isTimeSeries) { settings.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES) @@ -191,15 +192,15 @@ private void createIndex(String index, String alias, boolean isTimeSeries) throw createIndexWithSettings(client(), index, alias, settings, mapping); } - @TestLogging(value = "org.elasticsearch.xpack.ilm:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/105437") public void testRollupIndex() throws Exception { - createIndex(index, alias, true); - index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); - + // Create the ILM policy String phaseName = randomFrom("warm", "cold"); DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); createNewSingletonPolicy(client(), policy, phaseName, new DownsampleAction(fixedInterval, DownsampleAction.DEFAULT_WAIT_TIMEOUT)); - updatePolicy(client(), index, policy); + + // Create a time series index managed by the policy + createIndex(index, alias, policy, true); + index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); String rollupIndex = waitAndGetRollupIndexName(client(), index, fixedInterval); assertNotNull("Cannot retrieve rollup index name", rollupIndex); @@ -222,10 +223,7 @@ public void testRollupIndex() throws Exception { ); } - public void testRollupIndexInTheHotPhase() throws Exception { - createIndex(index, alias, true); - index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); - + public void testRollupIndexInTheHotPhaseWithoutRollover() { ResponseException e = expectThrows( ResponseException.class, () -> createNewSingletonPolicy( @@ -274,7 +272,7 @@ public void testRollupIndexInTheHotPhaseAfterRollover() throws Exception { client().performRequest(createTemplateRequest); // then create the index and index a document to trigger rollover - createIndex(originalIndex, alias, true); + createIndex(originalIndex, alias, policy, true); index( client(), originalIndex, @@ -396,15 +394,15 @@ public void testILMWaitsForTimeSeriesEndTimeToLapse() throws Exception { }, 30, TimeUnit.SECONDS); } - @TestLogging(value = "org.elasticsearch.xpack.ilm:TRACE", reason = "https://github.com/elastic/elasticsearch/issues/103981") public void testRollupNonTSIndex() throws Exception { - createIndex(index, alias, false); - index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); - + // Create the ILM policy String phaseName = randomFrom("warm", "cold"); DateHistogramInterval fixedInterval = ConfigTestHelpers.randomInterval(); createNewSingletonPolicy(client(), policy, phaseName, new DownsampleAction(fixedInterval, DownsampleAction.DEFAULT_WAIT_TIMEOUT)); - updatePolicy(client(), index, policy); + + // Create a non TSDB managed index + createIndex(index, alias, policy, false); + index(client(), index, true, null, "@timestamp", "2020-01-01T05:10:00Z", "volume", 11.0, "metricset", randomAlphaOfLength(5)); try { assertBusy( diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java index 8cc14a42eb5f3..472b9bdd0b800 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/action/TransportMigrateToDataTiersAction.java @@ -123,9 +123,9 @@ protected void masterOperation( final SetOnce migratedEntities = new SetOnce<>(); submitUnbatchedTask("migrate-to-data-tiers []", new ClusterStateUpdateTask(Priority.HIGH) { @Override - public ClusterState execute(ClusterState currentState) throws Exception { + public ClusterState execute(ClusterState currentState) { Tuple migratedEntitiesTuple = migrateToDataTiersRouting( - state, + currentState, request.getNodeAttributeName(), request.getLegacyTemplateToDelete(), xContentRegistry, diff --git a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java index 27fa55b7b7dc0..7d5c21b78ee8a 100644 --- a/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java +++ b/x-pack/plugin/inference/qa/test-service-plugin/src/main/java/org/elasticsearch/xpack/inference/mock/TestSparseInferenceServiceExtension.java @@ -30,7 +30,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import java.io.IOException; @@ -147,7 +147,7 @@ private List makeChunkedResults(List inp } results.add( new InferenceChunkedSparseEmbeddingResults( - List.of(new InferenceChunkedTextExpansionResults.ChunkedResult(input.get(i), tokens)) + List.of(new MlChunkedTextExpansionResults.ChunkedResult(input.get(i), tokens)) ) ); } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java index 573e77a58991c..f1a590e647dbc 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java @@ -392,7 +392,7 @@ private void applyInferenceResponses(BulkItemRequest item, FieldInferenceRespons new SemanticTextField.InferenceResult( model.getInferenceEntityId(), new SemanticTextField.ModelSettings(model), - toSemanticTextFieldChunks(fieldName, model.getInferenceEntityId(), results, indexRequest.getContentType()) + toSemanticTextFieldChunks(results, indexRequest.getContentType()) ), indexRequest.getContentType() ); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunker.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunker.java index 78a7522448464..01a345909c6b1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunker.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunker.java @@ -10,11 +10,14 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.common.util.concurrent.AtomicArray; +import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.inference.InferenceServiceResults; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import java.util.ArrayList; @@ -35,6 +38,18 @@ */ public class EmbeddingRequestChunker { + public enum EmbeddingType { + FLOAT, + BYTE; + + public static EmbeddingType fromDenseVectorElementType(DenseVectorFieldMapper.ElementType elementType) { + return switch (elementType) { + case BYTE -> EmbeddingType.BYTE; + case FLOAT -> EmbeddingType.FLOAT; + }; + } + }; + public static final int DEFAULT_WORDS_PER_CHUNK = 250; public static final int DEFAULT_CHUNK_OVERLAP = 100; @@ -43,37 +58,49 @@ public class EmbeddingRequestChunker { private final int maxNumberOfInputsPerBatch; private final int wordsPerChunk; private final int chunkOverlap; + private final EmbeddingType embeddingType; private List> chunkedInputs; - private List>> results; + private List>> floatResults; + private List>> byteResults; private AtomicArray errors; private ActionListener> finalListener; - public EmbeddingRequestChunker(List inputs, int maxNumberOfInputsPerBatch) { - this.maxNumberOfInputsPerBatch = maxNumberOfInputsPerBatch; - this.wordsPerChunk = DEFAULT_WORDS_PER_CHUNK; - this.chunkOverlap = DEFAULT_CHUNK_OVERLAP; - splitIntoBatchedRequests(inputs); + public EmbeddingRequestChunker(List inputs, int maxNumberOfInputsPerBatch, EmbeddingType embeddingType) { + this(inputs, maxNumberOfInputsPerBatch, DEFAULT_WORDS_PER_CHUNK, DEFAULT_CHUNK_OVERLAP, embeddingType); } - public EmbeddingRequestChunker(List inputs, int maxNumberOfInputsPerBatch, int wordsPerChunk, int chunkOverlap) { + public EmbeddingRequestChunker( + List inputs, + int maxNumberOfInputsPerBatch, + int wordsPerChunk, + int chunkOverlap, + EmbeddingType embeddingType + ) { this.maxNumberOfInputsPerBatch = maxNumberOfInputsPerBatch; this.wordsPerChunk = wordsPerChunk; this.chunkOverlap = chunkOverlap; + this.embeddingType = embeddingType; splitIntoBatchedRequests(inputs); } private void splitIntoBatchedRequests(List inputs) { var chunker = new WordBoundaryChunker(); chunkedInputs = new ArrayList<>(inputs.size()); - results = new ArrayList<>(inputs.size()); + switch (embeddingType) { + case FLOAT -> floatResults = new ArrayList<>(inputs.size()); + case BYTE -> byteResults = new ArrayList<>(inputs.size()); + } errors = new AtomicArray<>(inputs.size()); for (int i = 0; i < inputs.size(); i++) { var chunks = chunker.chunk(inputs.get(i), wordsPerChunk, chunkOverlap); int numberOfSubBatches = addToBatches(chunks, i); // size the results array with the expected number of request/responses - results.add(new AtomicArray<>(numberOfSubBatches)); + switch (embeddingType) { + case FLOAT -> floatResults.add(new AtomicArray<>(numberOfSubBatches)); + case BYTE -> byteResults.add(new AtomicArray<>(numberOfSubBatches)); + } chunkedInputs.add(chunks); } } @@ -160,33 +187,81 @@ private class DebatchingListener implements ActionListener handleFloatResults(inferenceServiceResults); + case BYTE -> handleByteResults(inferenceServiceResults); + } + } + + private void handleFloatResults(InferenceServiceResults inferenceServiceResults) { + if (inferenceServiceResults instanceof InferenceTextEmbeddingFloatResults floatEmbeddings) { + if (failIfNumRequestsDoNotMatch(floatEmbeddings.embeddings().size())) { return; } int start = 0; for (var pos : positions) { - results.get(pos.inputIndex()) - .setOnce(pos.chunkIndex(), textEmbeddingResults.embeddings().subList(start, start + pos.embeddingCount())); + floatResults.get(pos.inputIndex()) + .setOnce(pos.chunkIndex(), floatEmbeddings.embeddings().subList(start, start + pos.embeddingCount())); start += pos.embeddingCount(); } + + if (resultCount.incrementAndGet() == totalNumberOfRequests) { + sendResponse(); + } + } else { + onFailure( + unexpectedResultTypeException(inferenceServiceResults.getWriteableName(), InferenceTextEmbeddingFloatResults.NAME) + ); } + } - if (resultCount.incrementAndGet() == totalNumberOfRequests) { - sendResponse(); + private void handleByteResults(InferenceServiceResults inferenceServiceResults) { + if (inferenceServiceResults instanceof InferenceTextEmbeddingByteResults byteEmbeddings) { + if (failIfNumRequestsDoNotMatch(byteEmbeddings.embeddings().size())) { + return; + } + + int start = 0; + for (var pos : positions) { + byteResults.get(pos.inputIndex()) + .setOnce(pos.chunkIndex(), byteEmbeddings.embeddings().subList(start, start + pos.embeddingCount())); + start += pos.embeddingCount(); + } + + if (resultCount.incrementAndGet() == totalNumberOfRequests) { + sendResponse(); + } + } else { + onFailure( + unexpectedResultTypeException(inferenceServiceResults.getWriteableName(), InferenceTextEmbeddingByteResults.NAME) + ); } } + private boolean failIfNumRequestsDoNotMatch(int numberOfResults) { + int numberOfRequests = positions.stream().mapToInt(SubBatchPositionsAndCount::embeddingCount).sum(); + if (numberOfRequests != numberOfResults) { + onFailure( + new ElasticsearchStatusException( + "Error the number of embedding responses [{}] does not equal the number of " + "requests [{}]", + RestStatus.INTERNAL_SERVER_ERROR, + numberOfResults, + numberOfRequests + ) + ); + return true; + } + return false; + } + + private ElasticsearchStatusException unexpectedResultTypeException(String got, String expected) { + return new ElasticsearchStatusException( + "Unexpected inference result type [" + got + "], expected a [" + expected + "]", + RestStatus.INTERNAL_SERVER_ERROR + ); + } + @Override public void onFailure(Exception e) { var errorResult = new ErrorChunkedInferenceResults(e); @@ -205,34 +280,63 @@ private void sendResponse() { if (errors.get(i) != null) { response.add(errors.get(i)); } else { - response.add(merge(chunkedInputs.get(i), results.get(i))); + response.add(mergeResultsWithInputs(i)); } } finalListener.onResponse(response); } + } - private InferenceChunkedTextEmbeddingFloatResults merge( - List chunks, - AtomicArray> debatchedResults - ) { - var all = new ArrayList(); - for (int i = 0; i < debatchedResults.length(); i++) { - var subBatch = debatchedResults.get(i); - all.addAll(subBatch); - } + private ChunkedInferenceServiceResults mergeResultsWithInputs(int resultIndex) { + return switch (embeddingType) { + case FLOAT -> mergeFloatResultsWithInputs(chunkedInputs.get(resultIndex), floatResults.get(resultIndex)); + case BYTE -> mergeByteResultsWithInputs(chunkedInputs.get(resultIndex), byteResults.get(resultIndex)); + }; + } - assert chunks.size() == all.size(); + private InferenceChunkedTextEmbeddingFloatResults mergeFloatResultsWithInputs( + List chunks, + AtomicArray> debatchedResults + ) { + var all = new ArrayList(); + for (int i = 0; i < debatchedResults.length(); i++) { + var subBatch = debatchedResults.get(i); + all.addAll(subBatch); + } - var embeddingChunks = new ArrayList(); - for (int i = 0; i < chunks.size(); i++) { - embeddingChunks.add( - new InferenceChunkedTextEmbeddingFloatResults.InferenceFloatEmbeddingChunk(chunks.get(i), all.get(i).values()) - ); - } + assert chunks.size() == all.size(); + + var embeddingChunks = new ArrayList(); + for (int i = 0; i < chunks.size(); i++) { + embeddingChunks.add( + new InferenceChunkedTextEmbeddingFloatResults.InferenceFloatEmbeddingChunk(chunks.get(i), all.get(i).values()) + ); + } + + return new InferenceChunkedTextEmbeddingFloatResults(embeddingChunks); + } + + private InferenceChunkedTextEmbeddingByteResults mergeByteResultsWithInputs( + List chunks, + AtomicArray> debatchedResults + ) { + var all = new ArrayList(); + for (int i = 0; i < debatchedResults.length(); i++) { + var subBatch = debatchedResults.get(i); + all.addAll(subBatch); + } - return new InferenceChunkedTextEmbeddingFloatResults(embeddingChunks); + assert chunks.size() == all.size(); + + var embeddingChunks = new ArrayList(); + for (int i = 0; i < chunks.size(); i++) { + embeddingChunks.add( + new InferenceChunkedTextEmbeddingByteResults.InferenceByteEmbeddingChunk(chunks.get(i), all.get(i).values()) + ); } + + return new InferenceChunkedTextEmbeddingByteResults(embeddingChunks, false); } public record BatchRequest(List subBatches) { diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java index 7b2e23f2e972d..8ec614247bfbb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/mapper/SemanticTextField.java @@ -278,12 +278,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws /** * Converts the provided {@link ChunkedInferenceServiceResults} into a list of {@link Chunk}. */ - public static List toSemanticTextFieldChunks( - String field, - String inferenceId, - List results, - XContentType contentType - ) { + public static List toSemanticTextFieldChunks(List results, XContentType contentType) { List chunks = new ArrayList<>(); for (var result : results) { for (Iterator it = result.chunksAsMatchedTextAndByteReference(contentType.xContent()); it diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java index 214c652a97545..65c3db4093249 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioService.java @@ -24,10 +24,7 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; +import org.elasticsearch.xpack.inference.common.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.external.action.azureaistudio.AzureAiStudioActionCreator; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; @@ -44,7 +41,6 @@ import java.util.Map; import java.util.Set; -import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; @@ -53,6 +49,7 @@ import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioProviderCapabilities.providerAllowsEndpointTypeForTask; import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioProviderCapabilities.providerAllowsTaskType; import static org.elasticsearch.xpack.inference.services.azureaistudio.completion.AzureAiStudioChatCompletionTaskSettings.DEFAULT_MAX_NEW_TOKENS; +import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.EMBEDDING_MAX_BATCH_SIZE; public class AzureAiStudioService extends SenderService { @@ -105,23 +102,16 @@ protected void doChunkedInfer( TimeValue timeout, ActionListener> listener ) { - ActionListener inferListener = listener.delegateFailureAndWrap( - (delegate, response) -> delegate.onResponse(translateToChunkedResults(input, response)) - ); - - doInfer(model, input, taskSettings, inputType, timeout, inferListener); - } - - private static List translateToChunkedResults( - List inputs, - InferenceServiceResults inferenceResults - ) { - if (inferenceResults instanceof InferenceTextEmbeddingFloatResults textEmbeddingResults) { - return InferenceChunkedTextEmbeddingFloatResults.listOf(inputs, textEmbeddingResults); - } else if (inferenceResults instanceof ErrorInferenceResults error) { - return List.of(new ErrorChunkedInferenceResults(error.getException())); + if (model instanceof AzureAiStudioModel baseAzureAiStudioModel) { + var actionCreator = new AzureAiStudioActionCreator(getSender(), getServiceComponents()); + var batchedRequests = new EmbeddingRequestChunker(input, EMBEDDING_MAX_BATCH_SIZE, EmbeddingRequestChunker.EmbeddingType.FLOAT) + .batchRequestsWithListeners(listener); + for (var request : batchedRequests) { + var action = baseAzureAiStudioModel.accept(actionCreator, taskSettings); + action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); + } } else { - throw createInvalidChunkedResultException(InferenceTextEmbeddingFloatResults.NAME, inferenceResults.getWriteableName()); + listener.onFailure(createInvalidModelException(model)); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java index bd52bdb165148..5c25ae62517dd 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiService.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; +import org.elasticsearch.xpack.inference.common.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.external.action.azureopenai.AzureOpenAiActionCreator; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; @@ -49,6 +50,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrThrowIfNull; import static org.elasticsearch.xpack.inference.services.ServiceUtils.throwIfNotEmptyMap; +import static org.elasticsearch.xpack.inference.services.openai.OpenAiServiceFields.EMBEDDING_MAX_BATCH_SIZE; public class AzureOpenAiService extends SenderService { public static final String NAME = "azureopenai"; @@ -230,11 +232,18 @@ protected void doChunkedInfer( TimeValue timeout, ActionListener> listener ) { - ActionListener inferListener = listener.delegateFailureAndWrap( - (delegate, response) -> delegate.onResponse(translateToChunkedResults(input, response)) - ); - - doInfer(model, input, taskSettings, inputType, timeout, inferListener); + if (model instanceof AzureOpenAiModel == false) { + listener.onFailure(createInvalidModelException(model)); + return; + } + AzureOpenAiModel azureOpenAiModel = (AzureOpenAiModel) model; + var actionCreator = new AzureOpenAiActionCreator(getSender(), getServiceComponents()); + var batchedRequests = new EmbeddingRequestChunker(input, EMBEDDING_MAX_BATCH_SIZE, EmbeddingRequestChunker.EmbeddingType.FLOAT) + .batchRequestsWithListeners(listener); + for (var request : batchedRequests) { + var action = azureOpenAiModel.accept(actionCreator, taskSettings); + action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); + } } private static List translateToChunkedResults( diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java index 4c673026d7efb..76ef15568d448 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereService.java @@ -33,6 +33,7 @@ import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.ServiceUtils; import org.elasticsearch.xpack.inference.services.cohere.completion.CohereCompletionModel; +import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingType; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsModel; import org.elasticsearch.xpack.inference.services.cohere.embeddings.CohereEmbeddingsServiceSettings; import org.elasticsearch.xpack.inference.services.cohere.rerank.CohereRerankModel; @@ -247,7 +248,11 @@ protected void doChunkedInfer( CohereModel cohereModel = (CohereModel) model; var actionCreator = new CohereActionCreator(getSender(), getServiceComponents()); - var batchedRequests = new EmbeddingRequestChunker(input, EMBEDDING_MAX_BATCH_SIZE).batchRequestsWithListeners(listener); + var batchedRequests = new EmbeddingRequestChunker( + input, + EMBEDDING_MAX_BATCH_SIZE, + EmbeddingRequestChunker.EmbeddingType.fromDenseVectorElementType(model.getServiceSettings().elementType()) + ).batchRequestsWithListeners(listener); for (var request : batchedRequests) { var action = cohereModel.accept(actionCreator, taskSettings, inputType); action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); @@ -276,7 +281,9 @@ public void checkModelConfig(Model model, ActionListener listener) { private CohereEmbeddingsModel updateModelWithEmbeddingDetails(CohereEmbeddingsModel model, int embeddingSize) { var similarityFromModel = model.getServiceSettings().similarity(); - var similarityToUse = similarityFromModel == null ? SimilarityMeasure.DOT_PRODUCT : similarityFromModel; + var similarityToUse = similarityFromModel == null + ? defaultSimilarity(model.getServiceSettings().getEmbeddingType()) + : similarityFromModel; CohereEmbeddingsServiceSettings serviceSettings = new CohereEmbeddingsServiceSettings( new CohereServiceSettings( @@ -293,6 +300,29 @@ private CohereEmbeddingsModel updateModelWithEmbeddingDetails(CohereEmbeddingsMo return new CohereEmbeddingsModel(model, serviceSettings); } + /** + * Return the default similarity measure for the embedding type. + * Cohere embeddings are normalized to unit vectors so Dot Product + * can be used. However, Elasticsearch rejects the byte vectors with + * Dot Product similarity complaining they are not normalized so + * Cosine is used for bytes. + * TODO investigate why the byte vectors are not normalized. + * + * @param embeddingType The embedding type (can be null) + * @return The default similarity. + */ + static SimilarityMeasure defaultSimilarity(@Nullable CohereEmbeddingType embeddingType) { + if (embeddingType == null) { + return SimilarityMeasure.DOT_PRODUCT; + } + + return switch (embeddingType) { + case FLOAT -> SimilarityMeasure.DOT_PRODUCT; + case BYTE -> SimilarityMeasure.COSINE; + case INT8 -> SimilarityMeasure.COSINE; + }; + } + @Override public TransportVersion getMinimalSupportedVersion() { return TransportVersions.ML_INFERENCE_RATE_LIMIT_SETTINGS_ADDED; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java index ed0f1cd93c83e..11c97f8b8e37e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalService.java @@ -41,7 +41,7 @@ import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -415,7 +415,7 @@ private List translateChunkedResults(List(); for (var inferenceResult : inferenceResults) { - if (inferenceResult instanceof InferenceChunkedTextExpansionResults mlChunkedResult) { + if (inferenceResult instanceof MlChunkedTextExpansionResults mlChunkedResult) { translated.add(InferenceChunkedSparseEmbeddingResults.ofMlResult(mlChunkedResult)); } else if (inferenceResult instanceof ErrorInferenceResults error) { translated.add(new ErrorChunkedInferenceResults(error.getException())); @@ -423,7 +423,7 @@ private List translateChunkedResults(List input, - Map taskSettings, - InputType inputType, - ChunkingOptions chunkingOptions, - TimeValue timeout, - ActionListener> listener - ) { - ActionListener inferListener = listener.delegateFailureAndWrap( - (delegate, response) -> delegate.onResponse(translateToChunkedResults(input, response)) - ); - - doInfer(model, input, taskSettings, inputType, timeout, inferListener); - } - - private static List translateToChunkedResults( - List inputs, - InferenceServiceResults inferenceResults - ) { - if (inferenceResults instanceof InferenceTextEmbeddingFloatResults textEmbeddingResults) { - return InferenceChunkedTextEmbeddingFloatResults.listOf(inputs, textEmbeddingResults); - } else if (inferenceResults instanceof SparseEmbeddingResults sparseEmbeddingResults) { - return InferenceChunkedSparseEmbeddingResults.listOf(inputs, sparseEmbeddingResults); - } else if (inferenceResults instanceof ErrorInferenceResults error) { - return List.of(new ErrorChunkedInferenceResults(error.getException())); - } else { - String expectedClasses = Strings.format( - "One of [%s,%s]", - InferenceTextEmbeddingFloatResults.class.getSimpleName(), - SparseEmbeddingResults.class.getSimpleName() - ); - throw createInvalidChunkedResultException(expectedClasses, inferenceResults.getWriteableName()); - } - } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java index c0438b3759a65..161ab6c47bfeb 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceService.java @@ -12,9 +12,16 @@ import org.elasticsearch.TransportVersions; import org.elasticsearch.action.ActionListener; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.InputType; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.inference.common.EmbeddingRequestChunker; +import org.elasticsearch.xpack.inference.external.action.huggingface.HuggingFaceActionCreator; +import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceComponents; @@ -22,8 +29,11 @@ import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserModel; import org.elasticsearch.xpack.inference.services.huggingface.embeddings.HuggingFaceEmbeddingsModel; +import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; + public class HuggingFaceService extends HuggingFaceBaseService { public static final String NAME = "hugging_face"; @@ -79,6 +89,33 @@ private static HuggingFaceEmbeddingsModel updateModelWithEmbeddingDetails(Huggin return new HuggingFaceEmbeddingsModel(model, serviceSettings); } + @Override + protected void doChunkedInfer( + Model model, + @Nullable String query, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + TimeValue timeout, + ActionListener> listener + ) { + if (model instanceof HuggingFaceModel == false) { + listener.onFailure(createInvalidModelException(model)); + return; + } + + var huggingFaceModel = (HuggingFaceModel) model; + var actionCreator = new HuggingFaceActionCreator(getSender(), getServiceComponents()); + + var batchedRequests = new EmbeddingRequestChunker(input, EMBEDDING_MAX_BATCH_SIZE, EmbeddingRequestChunker.EmbeddingType.FLOAT) + .batchRequestsWithListeners(listener); + for (var request : batchedRequests) { + var action = huggingFaceModel.accept(actionCreator); + action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); + } + } + @Override public String name() { return NAME; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java index d3099e96ee7c1..ee35869c6a8d1 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/huggingface/elser/HuggingFaceElserService.java @@ -10,17 +10,34 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.TransportVersion; import org.elasticsearch.TransportVersions; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.InferenceServiceResults; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.inference.Model; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.ServiceComponents; import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceBaseService; import org.elasticsearch.xpack.inference.services.huggingface.HuggingFaceModel; +import java.util.List; import java.util.Map; +import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException; + public class HuggingFaceElserService extends HuggingFaceBaseService { public static final String NAME = "hugging_face_elser"; @@ -48,6 +65,45 @@ protected HuggingFaceModel createModel( }; } + @Override + protected void doChunkedInfer( + Model model, + @Nullable String query, + List input, + Map taskSettings, + InputType inputType, + ChunkingOptions chunkingOptions, + TimeValue timeout, + ActionListener> listener + ) { + ActionListener inferListener = listener.delegateFailureAndWrap( + (delegate, response) -> delegate.onResponse(translateToChunkedResults(input, response)) + ); + + // TODO chunking sparse embeddings not implemented + doInfer(model, input, taskSettings, inputType, timeout, inferListener); + } + + private static List translateToChunkedResults( + List inputs, + InferenceServiceResults inferenceResults + ) { + if (inferenceResults instanceof InferenceTextEmbeddingFloatResults textEmbeddingResults) { + return InferenceChunkedTextEmbeddingFloatResults.listOf(inputs, textEmbeddingResults); + } else if (inferenceResults instanceof SparseEmbeddingResults sparseEmbeddingResults) { + return InferenceChunkedSparseEmbeddingResults.listOf(inputs, sparseEmbeddingResults); + } else if (inferenceResults instanceof ErrorInferenceResults error) { + return List.of(new ErrorChunkedInferenceResults(error.getException())); + } else { + String expectedClasses = Strings.format( + "One of [%s,%s]", + InferenceTextEmbeddingFloatResults.class.getSimpleName(), + SparseEmbeddingResults.class.getSimpleName() + ); + throw createInvalidChunkedResultException(expectedClasses, inferenceResults.getWriteableName()); + } + } + @Override public TransportVersion getMinimalSupportedVersion() { return TransportVersions.V_8_12_0; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java index ee0cec1d75846..bcef31031cb0c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/MistralService.java @@ -22,10 +22,6 @@ import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; import org.elasticsearch.xpack.inference.common.EmbeddingRequestChunker; import org.elasticsearch.xpack.inference.external.action.mistral.MistralActionCreator; import org.elasticsearch.xpack.inference.external.http.sender.DocumentsOnlyInput; @@ -42,7 +38,6 @@ import java.util.Set; import static org.elasticsearch.TransportVersions.ADD_MISTRAL_EMBEDDINGS_INFERENCE; -import static org.elasticsearch.xpack.core.inference.results.ResultUtils.createInvalidChunkedResultException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.createInvalidModelException; import static org.elasticsearch.xpack.inference.services.ServiceUtils.parsePersistedConfigErrorMsg; import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeFromMapOrDefaultEmpty; @@ -102,7 +97,11 @@ protected void doChunkedInfer( var actionCreator = new MistralActionCreator(getSender(), getServiceComponents()); if (model instanceof MistralEmbeddingsModel mistralEmbeddingsModel) { - var batchedRequests = new EmbeddingRequestChunker(input, MistralConstants.MAX_BATCH_SIZE).batchRequestsWithListeners(listener); + var batchedRequests = new EmbeddingRequestChunker( + input, + MistralConstants.MAX_BATCH_SIZE, + EmbeddingRequestChunker.EmbeddingType.FLOAT + ).batchRequestsWithListeners(listener); for (var request : batchedRequests) { var action = mistralEmbeddingsModel.accept(actionCreator, taskSettings); @@ -113,19 +112,6 @@ protected void doChunkedInfer( } } - private static List translateToChunkedResults( - List inputs, - InferenceServiceResults inferenceResults - ) { - if (inferenceResults instanceof InferenceTextEmbeddingFloatResults textEmbeddingResults) { - return InferenceChunkedTextEmbeddingFloatResults.listOf(inputs, textEmbeddingResults); - } else if (inferenceResults instanceof ErrorInferenceResults error) { - return List.of(new ErrorChunkedInferenceResults(error.getException())); - } else { - throw createInvalidChunkedResultException(InferenceChunkedTextEmbeddingFloatResults.NAME, inferenceResults.getWriteableName()); - } - } - @Override public String name() { return NAME; diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java index c3d261efea79a..2631dfecccab3 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/mistral/embeddings/MistralEmbeddingsModel.java @@ -23,7 +23,6 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.Map; -import java.util.Objects; import static org.elasticsearch.xpack.inference.services.mistral.MistralConstants.API_EMBEDDINGS_PATH; @@ -51,23 +50,11 @@ public MistralEmbeddingsModel( ); } - public MistralEmbeddingsModel(MistralEmbeddingsModel model, TaskSettings taskSettings, RateLimitSettings rateLimitSettings) { - super(model, taskSettings); - this.model = Objects.requireNonNull(model.model); - this.rateLimitSettings = Objects.requireNonNull(rateLimitSettings); - setEndpointUrl(); - } - public MistralEmbeddingsModel(MistralEmbeddingsModel model, MistralEmbeddingsServiceSettings serviceSettings) { super(model, serviceSettings); setPropertiesFromServiceSettings(serviceSettings); } - protected MistralEmbeddingsModel(ModelConfigurations modelConfigurations, ModelSecrets modelSecrets) { - super(modelConfigurations, modelSecrets); - setPropertiesFromServiceSettings((MistralEmbeddingsServiceSettings) modelConfigurations.getServiceSettings()); - } - public MistralEmbeddingsModel( String inferenceEntityId, TaskType taskType, diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java index 04b6ae94d6b53..8e25d4a8936ab 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiService.java @@ -238,7 +238,8 @@ protected void doChunkedInfer( OpenAiModel openAiModel = (OpenAiModel) model; var actionCreator = new OpenAiActionCreator(getSender(), getServiceComponents()); - var batchedRequests = new EmbeddingRequestChunker(input, EMBEDDING_MAX_BATCH_SIZE).batchRequestsWithListeners(listener); + var batchedRequests = new EmbeddingRequestChunker(input, EMBEDDING_MAX_BATCH_SIZE, EmbeddingRequestChunker.EmbeddingType.FLOAT) + .batchRequestsWithListeners(listener); for (var request : batchedRequests) { var action = openAiModel.accept(actionCreator, taskSettings); action.execute(new DocumentsOnlyInput(request.batch().inputs()), timeout, request.listener()); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceFields.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceFields.java index ca2bc56866aa5..38b31dbe4bc22 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceFields.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/openai/OpenAiServiceFields.java @@ -16,6 +16,6 @@ public class OpenAiServiceFields { /** * Taken from https://platform.openai.com/docs/api-reference/embeddings/create */ - static final int EMBEDDING_MAX_BATCH_SIZE = 2048; + public static final int EMBEDDING_MAX_BATCH_SIZE = 2048; } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelConfigurationsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelConfigurationsTests.java index d52595a5899a8..5afae297b3592 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelConfigurationsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelConfigurationsTests.java @@ -20,8 +20,7 @@ public class ModelConfigurationsTests extends AbstractWireSerializingTestCase { public static ModelConfigurations createRandomInstance() { - // TODO randomise task types and settings - var taskType = TaskType.SPARSE_EMBEDDING; + var taskType = randomFrom(TaskType.values()); return new ModelConfigurations( randomAlphaOfLength(6), taskType, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java index ac7fc6ba56952..d6d139190c12c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/ModelSecretsTests.java @@ -27,10 +27,6 @@ public static ModelSecrets createRandomInstance() { return new ModelSecrets(randomSecretSettings()); } - public static ModelSecrets mutateTestInstance(ModelSecrets instance) { - return createRandomInstance(); - } - private static SecretSettings randomSecretSettings() { return new FakeSecretSettings(randomAlphaOfLengthBetween(8, 10)); } @@ -54,7 +50,7 @@ protected ModelSecrets createTestInstance() { @Override protected ModelSecrets mutateInstance(ModelSecrets instance) { - return mutateTestInstance(instance); + return randomValueOtherThan(instance, ModelSecretsTests::createRandomInstance); } public record FakeSecretSettings(String apiKey) implements SecretSettings { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/PutInferenceModelResponseTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/PutInferenceModelResponseTests.java index 89bd0247a9ccf..d88aa91ff5148 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/PutInferenceModelResponseTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/PutInferenceModelResponseTests.java @@ -23,8 +23,10 @@ protected PutInferenceModelAction.Response createTestInstance() { @Override protected PutInferenceModelAction.Response mutateInstance(PutInferenceModelAction.Response instance) { - var mutatedModel = ModelConfigurationsTests.mutateTestInstance(instance.getModel()); - return new PutInferenceModelAction.Response(mutatedModel); + return randomValueOtherThan(instance, () -> { + var mutatedModel = ModelConfigurationsTests.mutateTestInstance(instance.getModel()); + return new PutInferenceModelAction.Response(mutatedModel); + }); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunkerTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunkerTests.java index 66079a00ee3b8..facd8dfd9f3b1 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunkerTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/common/EmbeddingRequestChunkerTests.java @@ -11,7 +11,9 @@ import org.elasticsearch.inference.ChunkedInferenceServiceResults; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.inference.results.ErrorChunkedInferenceResults; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; +import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceTextEmbeddingFloatResults; import java.util.ArrayList; @@ -27,16 +29,18 @@ public class EmbeddingRequestChunkerTests extends ESTestCase { public void testShortInputsAreSingleBatch() { String input = "one chunk"; + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(List.of(input), 100, 100, 10).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(List.of(input), 100, 100, 10, embeddingType).batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); assertThat(batches.get(0).batch().inputs(), contains(input)); } public void testMultipleShortInputsAreSingleBatch() { List inputs = List.of("1st small", "2nd small", "3rd small"); + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(inputs, 100, 100, 10).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(inputs, 100, 100, 10, embeddingType).batchRequestsWithListeners(testListener()); assertThat(batches, hasSize(1)); assertEquals(batches.get(0).batch().inputs(), inputs); var subBatches = batches.get(0).batch().subBatches(); @@ -57,8 +61,11 @@ public void testManyInputsMakeManyBatches() { for (int i = 0; i < numInputs; i++) { inputs.add("input " + i); } + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(inputs, maxNumInputsPerBatch, 100, 10).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(inputs, maxNumInputsPerBatch, 100, 10, embeddingType).batchRequestsWithListeners( + testListener() + ); assertThat(batches, hasSize(4)); assertThat(batches.get(0).batch().inputs(), hasSize(maxNumInputsPerBatch)); assertThat(batches.get(1).batch().inputs(), hasSize(maxNumInputsPerBatch)); @@ -101,8 +108,11 @@ public void testLongInputChunkedOverMultipleBatches() { } List inputs = List.of("1st small", passageBuilder.toString(), "2nd small", "3rd small"); + var embeddingType = randomFrom(EmbeddingRequestChunker.EmbeddingType.values()); - var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(testListener()); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, embeddingType).batchRequestsWithListeners( + testListener() + ); assertThat(batches, hasSize(2)); { var batch = batches.get(0).batch(); @@ -157,7 +167,7 @@ public void testLongInputChunkedOverMultipleBatches() { } } - public void testMergingListener() { + public void testMergingListener_Float() { int batchSize = 5; int chunkSize = 20; int overlap = 0; @@ -172,7 +182,8 @@ public void testMergingListener() { List inputs = List.of("1st small", passageBuilder.toString(), "2nd small", "3rd small"); var finalListener = testListener(); - var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap).batchRequestsWithListeners(finalListener); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, EmbeddingRequestChunker.EmbeddingType.FLOAT) + .batchRequestsWithListeners(finalListener); assertThat(batches, hasSize(2)); // 4 inputs in 2 batches @@ -229,6 +240,79 @@ public void testMergingListener() { } } + public void testMergingListener_Byte() { + int batchSize = 5; + int chunkSize = 20; + int overlap = 0; + // passage will be chunked into batchSize + 1 parts + // and spread over 2 batch requests + int numberOfWordsInPassage = (chunkSize * batchSize) + 5; + + var passageBuilder = new StringBuilder(); + for (int i = 0; i < numberOfWordsInPassage; i++) { + passageBuilder.append("passage_input").append(i).append(" "); // chunk on whitespace + } + List inputs = List.of("1st small", passageBuilder.toString(), "2nd small", "3rd small"); + + var finalListener = testListener(); + var batches = new EmbeddingRequestChunker(inputs, batchSize, chunkSize, overlap, EmbeddingRequestChunker.EmbeddingType.BYTE) + .batchRequestsWithListeners(finalListener); + assertThat(batches, hasSize(2)); + + // 4 inputs in 2 batches + { + var embeddings = new ArrayList(); + for (int i = 0; i < batchSize; i++) { + embeddings.add(new InferenceTextEmbeddingByteResults.InferenceByteEmbedding(new byte[] { randomByte() })); + } + batches.get(0).listener().onResponse(new InferenceTextEmbeddingByteResults(embeddings)); + } + { + var embeddings = new ArrayList(); + for (int i = 0; i < 4; i++) { // 4 requests in the 2nd batch + embeddings.add(new InferenceTextEmbeddingByteResults.InferenceByteEmbedding(new byte[] { randomByte() })); + } + batches.get(1).listener().onResponse(new InferenceTextEmbeddingByteResults(embeddings)); + } + + assertNotNull(finalListener.results); + assertThat(finalListener.results, hasSize(4)); + { + var chunkedResult = finalListener.results.get(0); + assertThat(chunkedResult, instanceOf(InferenceChunkedTextEmbeddingByteResults.class)); + var chunkedByteResult = (InferenceChunkedTextEmbeddingByteResults) chunkedResult; + assertThat(chunkedByteResult.chunks(), hasSize(1)); + assertEquals("1st small", chunkedByteResult.chunks().get(0).matchedText()); + } + { + // this is the large input split in multiple chunks + var chunkedResult = finalListener.results.get(1); + assertThat(chunkedResult, instanceOf(InferenceChunkedTextEmbeddingByteResults.class)); + var chunkedByteResult = (InferenceChunkedTextEmbeddingByteResults) chunkedResult; + assertThat(chunkedByteResult.chunks(), hasSize(6)); + assertThat(chunkedByteResult.chunks().get(0).matchedText(), startsWith("passage_input0 ")); + assertThat(chunkedByteResult.chunks().get(1).matchedText(), startsWith(" passage_input20 ")); + assertThat(chunkedByteResult.chunks().get(2).matchedText(), startsWith(" passage_input40 ")); + assertThat(chunkedByteResult.chunks().get(3).matchedText(), startsWith(" passage_input60 ")); + assertThat(chunkedByteResult.chunks().get(4).matchedText(), startsWith(" passage_input80 ")); + assertThat(chunkedByteResult.chunks().get(5).matchedText(), startsWith(" passage_input100 ")); + } + { + var chunkedResult = finalListener.results.get(2); + assertThat(chunkedResult, instanceOf(InferenceChunkedTextEmbeddingByteResults.class)); + var chunkedByteResult = (InferenceChunkedTextEmbeddingByteResults) chunkedResult; + assertThat(chunkedByteResult.chunks(), hasSize(1)); + assertEquals("2nd small", chunkedByteResult.chunks().get(0).matchedText()); + } + { + var chunkedResult = finalListener.results.get(3); + assertThat(chunkedResult, instanceOf(InferenceChunkedTextEmbeddingByteResults.class)); + var chunkedByteResult = (InferenceChunkedTextEmbeddingByteResults) chunkedResult; + assertThat(chunkedByteResult.chunks(), hasSize(1)); + assertEquals("3rd small", chunkedByteResult.chunks().get(0).matchedText()); + } + } + public void testListenerErrorsWithWrongNumberOfResponses() { List inputs = List.of("1st small", "2nd small", "3rd small"); @@ -248,7 +332,8 @@ public void onFailure(Exception e) { } }; - var batches = new EmbeddingRequestChunker(inputs, 10, 100, 0).batchRequestsWithListeners(listener); + var batches = new EmbeddingRequestChunker(inputs, 10, 100, 0, EmbeddingRequestChunker.EmbeddingType.FLOAT) + .batchRequestsWithListeners(listener); assertThat(batches, hasSize(1)); var embeddings = new ArrayList(); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java index 51fa39b595a8e..6d8b3ab4fa28e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldTests.java @@ -20,7 +20,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import org.elasticsearch.xpack.core.utils.FloatConversionUtils; import org.elasticsearch.xpack.inference.model.TestModel; @@ -154,13 +154,13 @@ public static InferenceChunkedTextEmbeddingFloatResults randomInferenceChunkedTe } public static InferenceChunkedSparseEmbeddingResults randomSparseEmbeddings(List inputs) { - List chunks = new ArrayList<>(); + List chunks = new ArrayList<>(); for (String input : inputs) { var tokens = new ArrayList(); for (var token : input.split("\\s+")) { tokens.add(new WeightedToken(token, randomFloat())); } - chunks.add(new InferenceChunkedTextExpansionResults.ChunkedResult(input, tokens)); + chunks.add(new MlChunkedTextExpansionResults.ChunkedResult(input, tokens)); } return new InferenceChunkedSparseEmbeddingResults(chunks); } @@ -178,7 +178,7 @@ public static SemanticTextField randomSemanticText(String fieldName, Model model new SemanticTextField.InferenceResult( model.getInferenceEntityId(), new SemanticTextField.ModelSettings(model), - toSemanticTextFieldChunks(fieldName, model.getInferenceEntityId(), List.of(results), contentType) + toSemanticTextFieldChunks(List.of(results), contentType) ), contentType ); @@ -187,10 +187,10 @@ public static SemanticTextField randomSemanticText(String fieldName, Model model public static ChunkedInferenceServiceResults toChunkedResult(SemanticTextField field) throws IOException { switch (field.inference().modelSettings().taskType()) { case SPARSE_EMBEDDING -> { - List chunks = new ArrayList<>(); + List chunks = new ArrayList<>(); for (var chunk : field.inference().chunks()) { var tokens = parseWeightedTokens(chunk.rawEmbeddings(), field.contentType()); - chunks.add(new InferenceChunkedTextExpansionResults.ChunkedResult(chunk.text(), tokens)); + chunks.add(new MlChunkedTextExpansionResults.ChunkedResult(chunk.text(), tokens)); } return new InferenceChunkedSparseEmbeddingResults(chunks); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceChunkedSparseEmbeddingResultsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceChunkedSparseEmbeddingResultsTests.java index 9a2afdade296a..8685ad9f0e124 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceChunkedSparseEmbeddingResultsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/results/InferenceChunkedSparseEmbeddingResultsTests.java @@ -13,7 +13,7 @@ import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.SparseEmbeddingResults; import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.search.WeightedToken; import java.io.IOException; @@ -26,7 +26,7 @@ public class InferenceChunkedSparseEmbeddingResultsTests extends AbstractWireSerializingTestCase { public static InferenceChunkedSparseEmbeddingResults createRandomResults() { - var chunks = new ArrayList(); + var chunks = new ArrayList(); int numChunks = randomIntBetween(1, 5); for (int i = 0; i < numChunks; i++) { @@ -35,7 +35,7 @@ public static InferenceChunkedSparseEmbeddingResults createRandomResults() { for (int j = 0; j < numTokens; j++) { tokenWeights.add(new WeightedToken(Integer.toString(j), (float) randomDoubleBetween(0.0, 5.0, false))); } - chunks.add(new InferenceChunkedTextExpansionResults.ChunkedResult(randomAlphaOfLength(6), tokenWeights)); + chunks.add(new MlChunkedTextExpansionResults.ChunkedResult(randomAlphaOfLength(6), tokenWeights)); } return new InferenceChunkedSparseEmbeddingResults(chunks); @@ -43,7 +43,7 @@ public static InferenceChunkedSparseEmbeddingResults createRandomResults() { public void testToXContent_CreatesTheRightJsonForASingleChunk() { var entity = new InferenceChunkedSparseEmbeddingResults( - List.of(new InferenceChunkedTextExpansionResults.ChunkedResult("text", List.of(new WeightedToken("token", 0.1f)))) + List.of(new MlChunkedTextExpansionResults.ChunkedResult("text", List.of(new WeightedToken("token", 0.1f)))) ); assertThat( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java index 18d7b6e072fe3..709cc4d3494fd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/AzureAiStudioServiceTests.java @@ -33,7 +33,6 @@ import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.ChatCompletionResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -55,14 +54,12 @@ import org.junit.Before; import java.io.IOException; -import java.net.URISyntaxException; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResultsTests.asMapWithListsInsteadOfArrays; import static org.elasticsearch.xpack.inference.Utils.getInvalidModel; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; @@ -849,7 +846,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotAzureAiStudioModel() throws IOExc verifyNoMoreInteractions(sender); } - public void testChunkedInfer_Embeddings_CallsInfer_ConvertsFloatResponse() throws IOException, URISyntaxException { + public void testChunkedInfer() throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new AzureAiStudioService(senderFactory, createWithEmptySettings(threadPool))) { @@ -865,6 +862,14 @@ public void testChunkedInfer_Embeddings_CallsInfer_ConvertsFloatResponse() throw 0.0123, -0.0123 ] + }, + { + "object": "embedding", + "index": 1, + "embedding": [ + 1.0123, + -1.0123 + ] } ], "model": "text-embedding-ada-002-v2", @@ -892,7 +897,7 @@ public void testChunkedInfer_Embeddings_CallsInfer_ConvertsFloatResponse() throw PlainActionFuture> listener = new PlainActionFuture<>(); service.chunkedInfer( model, - List.of("abc"), + List.of("foo", "bar"), new HashMap<>(), InputType.INGEST, new ChunkingOptions(null, null), @@ -900,20 +905,23 @@ public void testChunkedInfer_Embeddings_CallsInfer_ConvertsFloatResponse() throw listener ); - var result = listener.actionGet(TIMEOUT).get(0); - assertThat(result, CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); + var results = listener.actionGet(TIMEOUT); + assertThat(results, hasSize(2)); + { + assertThat(results.get(0), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); + var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(0); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("foo", floatResult.chunks().get(0).matchedText()); + assertArrayEquals(new float[] { 0.0123f, -0.0123f }, floatResult.chunks().get(0).embedding(), 0.0f); + } + { + assertThat(results.get(1), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); + var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(1); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("bar", floatResult.chunks().get(0).matchedText()); + assertArrayEquals(new float[] { 1.0123f, -1.0123f }, floatResult.chunks().get(0).embedding(), 0.0f); + } - assertThat( - asMapWithListsInsteadOfArrays((InferenceChunkedTextEmbeddingFloatResults) result), - Matchers.is( - Map.of( - InferenceChunkedTextEmbeddingFloatResults.FIELD_NAME, - List.of( - Map.of(ChunkedNlpInferenceResults.TEXT, "abc", ChunkedNlpInferenceResults.INFERENCE, List.of(0.0123f, -0.0123f)) - ) - ) - ) - ); assertThat(webServer.requests(), hasSize(1)); assertNull(webServer.requests().get(0).getUri().getQuery()); assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); @@ -921,7 +929,7 @@ public void testChunkedInfer_Embeddings_CallsInfer_ConvertsFloatResponse() throw var requestMap = entityAsMap(webServer.requests().get(0).getBody()); assertThat(requestMap.size(), Matchers.is(2)); - assertThat(requestMap.get("input"), Matchers.is(List.of("abc"))); + assertThat(requestMap.get("input"), Matchers.is(List.of("foo", "bar"))); assertThat(requestMap.get("user"), Matchers.is("user")); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java index d46a5f190017a..95be365706ccb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureaistudio/completion/AzureAiStudioChatCompletionServiceSettingsTests.java @@ -7,15 +7,18 @@ package org.elasticsearch.xpack.inference.services.azureaistudio.completion; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.Strings; -import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ml.AbstractBWCWireSerializationTestCase; import org.elasticsearch.xpack.inference.services.ConfigurationParseContext; import org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioEndpointType; import org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioProvider; import org.elasticsearch.xpack.inference.services.settings.RateLimitSettings; +import org.elasticsearch.xpack.inference.services.settings.RateLimitSettingsTests; import org.hamcrest.CoreMatchers; import java.io.IOException; @@ -27,7 +30,8 @@ import static org.elasticsearch.xpack.inference.services.azureaistudio.AzureAiStudioConstants.TARGET_FIELD; import static org.hamcrest.Matchers.is; -public class AzureAiStudioChatCompletionServiceSettingsTests extends ESTestCase { +public class AzureAiStudioChatCompletionServiceSettingsTests extends AbstractBWCWireSerializationTestCase< + AzureAiStudioChatCompletionServiceSettings> { public void testFromMap_Request_CreatesSettingsCorrectly() { var target = "http://sometarget.local"; var provider = "openai"; @@ -119,4 +123,38 @@ public void testToFilteredXContent_WritesAllValues() throws IOException { public static HashMap createRequestSettingsMap(String target, String provider, String endpointType) { return new HashMap<>(Map.of(TARGET_FIELD, target, PROVIDER_FIELD, provider, ENDPOINT_TYPE_FIELD, endpointType)); } + + @Override + protected Writeable.Reader instanceReader() { + return AzureAiStudioChatCompletionServiceSettings::new; + } + + @Override + protected AzureAiStudioChatCompletionServiceSettings createTestInstance() { + return createRandom(); + } + + @Override + protected AzureAiStudioChatCompletionServiceSettings mutateInstance(AzureAiStudioChatCompletionServiceSettings instance) + throws IOException { + return randomValueOtherThan(instance, AzureAiStudioChatCompletionServiceSettingsTests::createRandom); + } + + @Override + protected AzureAiStudioChatCompletionServiceSettings mutateInstanceForVersion( + AzureAiStudioChatCompletionServiceSettings instance, + TransportVersion version + ) { + return instance; + } + + private static AzureAiStudioChatCompletionServiceSettings createRandom() { + return new AzureAiStudioChatCompletionServiceSettings( + randomAlphaOfLength(10), + randomFrom(AzureAiStudioProvider.values()), + randomFrom(AzureAiStudioEndpointType.values()), + RateLimitSettingsTests.createRandom() + ); + } + } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java index e59664d0e0129..de474ea1b4237 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/azureopenai/AzureOpenAiServiceTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; -import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; @@ -55,7 +54,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResultsTests.asMapWithListsInsteadOfArrays; import static org.elasticsearch.xpack.inference.Utils.getInvalidModel; import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; @@ -1079,8 +1077,16 @@ public void testChunkedInfer_CallsInfer_ConvertsFloatResponse() throws IOExcepti "object": "embedding", "index": 0, "embedding": [ - 0.0123, - -0.0123 + 0.123, + -0.123 + ] + }, + { + "object": "embedding", + "index": 1, + "embedding": [ + 1.123, + -1.123 ] } ], @@ -1098,7 +1104,7 @@ public void testChunkedInfer_CallsInfer_ConvertsFloatResponse() throws IOExcepti PlainActionFuture> listener = new PlainActionFuture<>(); service.chunkedInfer( model, - List.of("abc"), + List.of("foo", "bar"), new HashMap<>(), InputType.INGEST, new ChunkingOptions(null, null), @@ -1106,20 +1112,23 @@ public void testChunkedInfer_CallsInfer_ConvertsFloatResponse() throws IOExcepti listener ); - var result = listener.actionGet(TIMEOUT).get(0); - assertThat(result, CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); + var results = listener.actionGet(TIMEOUT); + assertThat(results, hasSize(2)); + { + assertThat(results.get(0), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); + var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(0); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("foo", floatResult.chunks().get(0).matchedText()); + assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + } + { + assertThat(results.get(1), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); + var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(1); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("bar", floatResult.chunks().get(0).matchedText()); + assertArrayEquals(new float[] { 1.123f, -1.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + } - assertThat( - asMapWithListsInsteadOfArrays((InferenceChunkedTextEmbeddingFloatResults) result), - Matchers.is( - Map.of( - InferenceChunkedTextEmbeddingFloatResults.FIELD_NAME, - List.of( - Map.of(ChunkedNlpInferenceResults.TEXT, "abc", ChunkedNlpInferenceResults.INFERENCE, List.of(0.0123f, -0.0123f)) - ) - ) - ) - ); assertThat(webServer.requests(), hasSize(1)); assertNull(webServer.requests().get(0).getUri().getQuery()); assertThat(webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), equalTo(XContentType.JSON.mediaType())); @@ -1127,7 +1136,7 @@ public void testChunkedInfer_CallsInfer_ConvertsFloatResponse() throws IOExcepti var requestMap = entityAsMap(webServer.requests().get(0).getBody()); assertThat(requestMap.size(), Matchers.is(2)); - assertThat(requestMap.get("input"), Matchers.is(List.of("abc"))); + assertThat(requestMap.get("input"), Matchers.is(List.of("foo", "bar"))); assertThat(requestMap.get("user"), Matchers.is("user")); } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java index 20eb183c17900..5b3cb9eade9de 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingByteResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSender; @@ -51,7 +52,6 @@ import org.junit.Before; import java.io.IOException; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -1225,14 +1225,14 @@ public void testChunkedInfer_BatchesCalls() throws IOException { var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(0); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("foo", floatResult.chunks().get(0).matchedText()); - assertTrue(Arrays.equals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding())); + assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f); } { assertThat(results.get(1), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(1); assertThat(floatResult.chunks(), hasSize(1)); assertEquals("bar", floatResult.chunks().get(0).matchedText()); - assertTrue(Arrays.equals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding())); + assertArrayEquals(new float[] { 0.223f, -0.223f }, floatResult.chunks().get(0).embedding(), 0.0f); } MatcherAssert.assertThat(webServer.requests(), hasSize(1)); @@ -1251,8 +1251,102 @@ public void testChunkedInfer_BatchesCalls() throws IOException { } } - public void testChunkedInfer_CallsInfer_ConvertsByteResponse() throws IOException { - // TODO byte response not implemented yet + public void testChunkedInfer_BatchesCalls_Bytes() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new CohereService(senderFactory, createWithEmptySettings(threadPool))) { + + // Batching will call the service with 2 inputs + String responseJson = """ + { + "id": "de37399c-5df6-47cb-bc57-e3c5680c977b", + "texts": [ + "hello" + ], + "embeddings": { + "int8": [ + [ + 23, + -23 + ], + [ + 24, + -24 + ] + ] + }, + "meta": { + "api_version": { + "version": "1" + }, + "billed_units": { + "input_tokens": 1 + } + }, + "response_type": "embeddings_by_type" + } + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = CohereEmbeddingsModelTests.createModel( + getUrl(webServer), + "secret", + new CohereEmbeddingsTaskSettings(null, null), + 1024, + 1024, + "model", + CohereEmbeddingType.BYTE + ); + PlainActionFuture> listener = new PlainActionFuture<>(); + // 2 inputs + service.chunkedInfer( + model, + List.of("foo", "bar"), + new HashMap<>(), + InputType.UNSPECIFIED, + new ChunkingOptions(null, null), + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var results = listener.actionGet(TIMEOUT); + assertThat(results, hasSize(2)); + { + assertThat(results.get(0), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingByteResults.class)); + var floatResult = (InferenceChunkedTextEmbeddingByteResults) results.get(0); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("foo", floatResult.chunks().get(0).matchedText()); + assertArrayEquals(new byte[] { 23, -23 }, floatResult.chunks().get(0).embedding()); + } + { + assertThat(results.get(1), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingByteResults.class)); + var byteResult = (InferenceChunkedTextEmbeddingByteResults) results.get(1); + assertThat(byteResult.chunks(), hasSize(1)); + assertEquals("bar", byteResult.chunks().get(0).matchedText()); + assertArrayEquals(new byte[] { 24, -24 }, byteResult.chunks().get(0).embedding()); + } + + MatcherAssert.assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + MatcherAssert.assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaType()) + ); + MatcherAssert.assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + MatcherAssert.assertThat( + requestMap, + is(Map.of("texts", List.of("foo", "bar"), "model", "model", "embedding_types", List.of("int8"))) + ); + } + } + + public void testDefaultSimilarity() { + assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity(null)); + assertEquals(SimilarityMeasure.DOT_PRODUCT, CohereService.defaultSimilarity(CohereEmbeddingType.FLOAT)); + assertEquals(SimilarityMeasure.COSINE, CohereService.defaultSimilarity(CohereEmbeddingType.INT8)); + assertEquals(SimilarityMeasure.COSINE, CohereService.defaultSimilarity(CohereEmbeddingType.BYTE)); } private Map getRequestConfigMap( diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java index 31962e44851c0..bc7dca4f11960 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/elser/ElserInternalServiceTests.java @@ -30,8 +30,8 @@ import org.elasticsearch.xpack.core.ml.action.PutTrainedModelAction; import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.results.ErrorInferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResultsTests; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TokenizationConfigUpdate; import org.junit.After; import org.junit.Before; @@ -383,10 +383,10 @@ public void testChunkInfer() { assertThat(chunkedResponse, hasSize(3)); assertThat(chunkedResponse.get(0), instanceOf(InferenceChunkedSparseEmbeddingResults.class)); var result1 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(0); - assertEquals(((InferenceChunkedTextExpansionResults) mlTrainedModelResults.get(0)).getChunks(), result1.getChunkedResults()); + assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(0)).getChunks(), result1.getChunkedResults()); assertThat(chunkedResponse.get(1), instanceOf(InferenceChunkedSparseEmbeddingResults.class)); var result2 = (InferenceChunkedSparseEmbeddingResults) chunkedResponse.get(1); - assertEquals(((InferenceChunkedTextExpansionResults) mlTrainedModelResults.get(1)).getChunks(), result2.getChunkedResults()); + assertEquals(((MlChunkedTextExpansionResults) mlTrainedModelResults.get(1)).getChunks(), result2.getChunkedResults()); var result3 = (ErrorChunkedInferenceResults) chunkedResponse.get(2); assertThat(result3.getException(), instanceOf(RuntimeException.class)); assertThat(result3.getException().getMessage(), containsString("boom")); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java index fd7e1b48b7e03..22c3b7895460a 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceBaseServiceTests.java @@ -90,7 +90,7 @@ public void testInfer_ThrowsErrorWhenModelIsNotHuggingFaceModel() throws IOExcep verifyNoMoreInteractions(sender); } - private static final class TestService extends HuggingFaceBaseService { + private static final class TestService extends HuggingFaceService { TestService(HttpRequestSender.Factory factory, ServiceComponents serviceComponents) { super(factory, serviceComponents); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java new file mode 100644 index 0000000000000..33ab75a543381 --- /dev/null +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceElserServiceTests.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.services.huggingface; + +import org.apache.http.HttpHeaders; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.inference.ChunkedInferenceServiceResults; +import org.elasticsearch.inference.ChunkingOptions; +import org.elasticsearch.inference.InputType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.http.MockResponse; +import org.elasticsearch.test.http.MockWebServer; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.inference.action.InferenceAction; +import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; +import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults; +import org.elasticsearch.xpack.inference.external.http.HttpClientManager; +import org.elasticsearch.xpack.inference.external.http.sender.HttpRequestSenderTests; +import org.elasticsearch.xpack.inference.logging.ThrottlerManager; +import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserModelTests; +import org.elasticsearch.xpack.inference.services.huggingface.elser.HuggingFaceElserService; +import org.hamcrest.MatcherAssert; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.elasticsearch.xpack.inference.Utils.inferenceUtilityPool; +import static org.elasticsearch.xpack.inference.Utils.mockClusterServiceEmpty; +import static org.elasticsearch.xpack.inference.external.http.Utils.entityAsMap; +import static org.elasticsearch.xpack.inference.external.http.Utils.getUrl; +import static org.elasticsearch.xpack.inference.services.ServiceComponentsTests.createWithEmptySettings; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; + +public class HuggingFaceElserServiceTests extends ESTestCase { + + private static final TimeValue TIMEOUT = new TimeValue(30, TimeUnit.SECONDS); + + private final MockWebServer webServer = new MockWebServer(); + private ThreadPool threadPool; + private HttpClientManager clientManager; + + @Before + public void init() throws Exception { + webServer.start(); + threadPool = createThreadPool(inferenceUtilityPool()); + clientManager = HttpClientManager.create(Settings.EMPTY, threadPool, mockClusterServiceEmpty(), mock(ThrottlerManager.class)); + } + + @After + public void shutdown() throws IOException { + clientManager.close(); + terminate(threadPool); + webServer.close(); + } + + public void testChunkedInfer_CallsInfer_Elser_ConvertsFloatResponse() throws IOException { + var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); + + try (var service = new HuggingFaceElserService(senderFactory, createWithEmptySettings(threadPool))) { + + String responseJson = """ + [ + { + ".": 0.133155956864357 + } + ] + """; + webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); + + var model = HuggingFaceElserModelTests.createModel(getUrl(webServer), "secret"); + PlainActionFuture> listener = new PlainActionFuture<>(); + service.chunkedInfer( + model, + List.of("abc"), + new HashMap<>(), + InputType.INGEST, + new ChunkingOptions(null, null), + InferenceAction.Request.DEFAULT_TIMEOUT, + listener + ); + + var result = listener.actionGet(TIMEOUT).get(0); + + MatcherAssert.assertThat( + result.asMap(), + Matchers.is( + Map.of( + InferenceChunkedSparseEmbeddingResults.FIELD_NAME, + List.of( + Map.of(ChunkedNlpInferenceResults.TEXT, "abc", ChunkedNlpInferenceResults.INFERENCE, Map.of(".", 0.13315596f)) + ) + ) + ) + ); + + assertThat(webServer.requests(), hasSize(1)); + assertNull(webServer.requests().get(0).getUri().getQuery()); + assertThat( + webServer.requests().get(0).getHeader(HttpHeaders.CONTENT_TYPE), + equalTo(XContentType.JSON.mediaTypeWithoutParameters()) + ); + assertThat(webServer.requests().get(0).getHeader(HttpHeaders.AUTHORIZATION), equalTo("Bearer secret")); + + var requestMap = entityAsMap(webServer.requests().get(0).getBody()); + assertThat(requestMap.size(), Matchers.is(1)); + assertThat(requestMap.get("inputs"), Matchers.is(List.of("abc"))); + } + } +} diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java index a36306e40f5cb..a855437ce0738 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/huggingface/HuggingFaceServiceTests.java @@ -31,7 +31,6 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.InferenceAction; -import org.elasticsearch.xpack.core.inference.results.InferenceChunkedSparseEmbeddingResults; import org.elasticsearch.xpack.core.inference.results.InferenceChunkedTextEmbeddingFloatResults; import org.elasticsearch.xpack.core.ml.inference.results.ChunkedNlpInferenceResults; import org.elasticsearch.xpack.inference.external.http.HttpClientManager; @@ -649,21 +648,22 @@ public void testChunkedInfer_CallsInfer_TextEmbedding_ConvertsFloatResponse() th } } - public void testChunkedInfer_CallsInfer_Elser_ConvertsFloatResponse() throws IOException { + public void testChunkedInfer() throws IOException { var senderFactory = HttpRequestSenderTests.createSenderFactory(threadPool, clientManager); try (var service = new HuggingFaceService(senderFactory, createWithEmptySettings(threadPool))) { String responseJson = """ [ - { - ".": 0.133155956864357 - } + [ + 0.123, + -0.123 + ] ] """; webServer.enqueue(new MockResponse().setResponseCode(200).setBody(responseJson)); - var model = HuggingFaceElserModelTests.createModel(getUrl(webServer), "secret"); + var model = HuggingFaceEmbeddingsModelTests.createModel(getUrl(webServer), "secret"); PlainActionFuture> listener = new PlainActionFuture<>(); service.chunkedInfer( model, @@ -675,19 +675,15 @@ public void testChunkedInfer_CallsInfer_Elser_ConvertsFloatResponse() throws IOE listener ); - var result = listener.actionGet(TIMEOUT).get(0); - - MatcherAssert.assertThat( - result.asMap(), - Matchers.is( - Map.of( - InferenceChunkedSparseEmbeddingResults.FIELD_NAME, - List.of( - Map.of(ChunkedNlpInferenceResults.TEXT, "abc", ChunkedNlpInferenceResults.INFERENCE, Map.of(".", 0.13315596f)) - ) - ) - ) - ); + var results = listener.actionGet(TIMEOUT); + assertThat(results, hasSize(1)); + { + assertThat(results.get(0), CoreMatchers.instanceOf(InferenceChunkedTextEmbeddingFloatResults.class)); + var floatResult = (InferenceChunkedTextEmbeddingFloatResults) results.get(0); + assertThat(floatResult.chunks(), hasSize(1)); + assertEquals("abc", floatResult.chunks().get(0).matchedText()); + assertArrayEquals(new float[] { 0.123f, -0.123f }, floatResult.chunks().get(0).embedding(), 0.0f); + } assertThat(webServer.requests(), hasSize(1)); assertNull(webServer.requests().get(0).getUri().getQuery()); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 05c807cffdd35..e356fc2756c56 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.BlockSourceReader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IgnoreMalformedStoredValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperParsingException; @@ -209,7 +210,14 @@ public UnsignedLongFieldMapper build(MapperBuilderContext context) { metric.getValue(), indexMode ); - return new UnsignedLongFieldMapper(name(), fieldType, multiFieldsBuilder.build(this, context), copyTo, this); + return new UnsignedLongFieldMapper( + name(), + fieldType, + multiFieldsBuilder.build(this, context), + copyTo, + context.isSourceSynthetic(), + this + ); } } @@ -554,6 +562,7 @@ public MetricType getMetricType() { } } + private final boolean isSourceSynthetic; private final boolean indexed; private final boolean hasDocValues; private final boolean stored; @@ -570,9 +579,11 @@ private UnsignedLongFieldMapper( MappedFieldType mappedFieldType, MultiFields multiFields, CopyTo copyTo, + boolean isSourceSynthetic, Builder builder ) { super(simpleName, mappedFieldType, multiFields, copyTo); + this.isSourceSynthetic = isSourceSynthetic; this.indexed = builder.indexed.getValue(); this.hasDocValues = builder.hasDocValues.getValue(); this.stored = builder.stored.getValue(); @@ -623,6 +634,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } catch (IllegalArgumentException e) { if (ignoreMalformed.value() && parser.currentToken().isValue()) { context.addIgnoredField(mappedFieldType.name()); + if (isSourceSynthetic) { + // Save a copy of the field so synthetic source can load it + context.doc().add(IgnoreMalformedStoredValues.storedField(name(), context.parser())); + } return; } else { throw e; @@ -757,11 +772,6 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values" ); } - if (ignoreMalformed.value()) { - throw new IllegalArgumentException( - "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers" - ); - } if (copyTo.copyToFields().isEmpty() != true) { throw new IllegalArgumentException( "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to" diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java index dfc1fd23c30eb..95fe4f7a17244 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongScriptDocValues.java @@ -22,6 +22,7 @@ public long getValue() { @Override public Long get(int index) { throwIfEmpty(); + throwIfBeyondLength(index); return supplier.getInternal(index); } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index fc783ef92a112..753440cb0b789 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -11,7 +11,6 @@ import org.apache.lucene.index.IndexableField; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DocumentMapper; @@ -33,6 +32,8 @@ import java.util.Collections; import java.util.List; import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -362,8 +363,7 @@ private Number randomNumericValue() { @Override protected SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed) { - assumeFalse("unsigned_long doesn't support ignore_malformed with synthetic _source", ignoreMalformed); - return new NumberSyntheticSourceSupport(); + return new NumberSyntheticSourceSupport(ignoreMalformed); } @Override @@ -417,30 +417,57 @@ protected Function loadBlockExpected() { final class NumberSyntheticSourceSupport implements SyntheticSourceSupport { private final BigInteger nullValue = usually() ? null : BigInteger.valueOf(randomNonNegativeLong()); + private final boolean ignoreMalformedEnabled; + + NumberSyntheticSourceSupport(boolean ignoreMalformedEnabled) { + this.ignoreMalformedEnabled = ignoreMalformedEnabled; + } @Override public SyntheticSourceExample example(int maxVals) { if (randomBoolean()) { - Tuple v = generateValue(); - return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping); + Value v = generateValue(); + if (v.malformedOutput == null) { + return new SyntheticSourceExample(v.input, v.output, this::mapping); + } + return new SyntheticSourceExample(v.input, v.malformedOutput, null, this::mapping); } - List> values = randomList(1, maxVals, this::generateValue); - List in = values.stream().map(Tuple::v1).toList(); - List outList = values.stream().map(Tuple::v2).sorted().toList(); + List values = randomList(1, maxVals, this::generateValue); + List in = values.stream().map(Value::input).toList(); + + List outputFromDocValues = values.stream() + .filter(v -> v.malformedOutput == null) + .map(Value::output) + .sorted() + .toList(); + Stream malformedOutput = values.stream().filter(v -> v.malformedOutput != null).map(Value::malformedOutput); + + // Malformed values are always last in the implementation. + List outList = Stream.concat(outputFromDocValues.stream(), malformedOutput).toList(); Object out = outList.size() == 1 ? outList.get(0) : outList; - return new SyntheticSourceExample(in, out, this::mapping); + + Object outBlock = outputFromDocValues.size() == 1 ? outputFromDocValues.get(0) : outputFromDocValues; + + return new SyntheticSourceExample(in, out, outBlock, this::mapping); } - private Tuple generateValue() { + private record Value(Object input, BigInteger output, Object malformedOutput) {} + + private Value generateValue() { if (nullValue != null && randomBoolean()) { - return Tuple.tuple(null, nullValue); + return new Value(null, nullValue, null); + } + if (ignoreMalformedEnabled && randomBoolean()) { + List> choices = List.of(() -> randomAlphaOfLengthBetween(1, 10)); + var malformedInput = randomFrom(choices).get(); + return new Value(malformedInput, null, malformedInput); } long n = randomNonNegativeLong(); BigInteger b = BigInteger.valueOf(n); if (b.signum() < 0) { b = b.add(BigInteger.ONE.shiftLeft(64)); } - return Tuple.tuple(n, b); + return new Value(n, b, null); } private void mapping(XContentBuilder b) throws IOException { @@ -454,6 +481,9 @@ private void mapping(XContentBuilder b) throws IOException { if (rarely()) { b.field("store", false); } + if (ignoreMalformedEnabled) { + b.field("ignore_malformed", "true"); + } } @Override @@ -465,13 +495,6 @@ public List invalidExample() { minimalMapping(b); b.field("doc_values", false); } - ), - new SyntheticSourceInvalidExample( - matchesPattern("field \\[field] of type \\[.+] doesn't support synthetic source because it ignores malformed numbers"), - b -> { - minimalMapping(b); - b.field("ignore_malformed", true); - } ) ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java index ab7c0e2ef53cc..de93a41fb7296 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartTrainedModelDeploymentAction.java @@ -45,6 +45,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.common.time.RemainingTime; import org.elasticsearch.xpack.core.inference.action.GetInferenceModelAction; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; @@ -70,6 +71,7 @@ import org.elasticsearch.xpack.ml.process.MlMemoryTracker; import org.elasticsearch.xpack.ml.utils.TaskRetriever; +import java.time.Instant; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -137,6 +139,7 @@ protected void masterOperation( ClusterState state, ActionListener listener ) throws Exception { + var remainingTime = RemainingTime.from(Instant::now, request.getTimeout()); logger.debug(() -> "[" + request.getDeploymentId() + "] received deploy request for model [" + request.getModelId() + "]"); if (MachineLearningField.ML_API_FEATURE.check(licenseState) == false) { listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING)); @@ -181,7 +184,7 @@ protected void masterOperation( AtomicLong perAllocationMemoryBytes = new AtomicLong(); ActionListener waitForDeploymentToStart = ActionListener.wrap( - modelAssignment -> waitForDeploymentState(request.getDeploymentId(), request.getTimeout(), request.getWaitForState(), listener), + modelAssignment -> waitForDeploymentState(request, remainingTime.get(), listener), e -> { logger.warn( () -> "[" + request.getDeploymentId() + "] creating new assignment for model [" + request.getModelId() + "] failed", @@ -268,7 +271,7 @@ protected void masterOperation( error -> { if (ExceptionsHelper.unwrapCause(error) instanceof ResourceNotFoundException) { // no name clash, continue with the deployment - checkFullModelDefinitionIsPresent(client, trainedModelConfig, true, request.getTimeout(), modelSizeListener); + checkFullModelDefinitionIsPresent(client, trainedModelConfig, true, remainingTime.get(), modelSizeListener); } else { listener.onFailure(error); } @@ -280,7 +283,7 @@ protected void masterOperation( if (request.getModelId().equals(request.getDeploymentId()) == false) { client.execute(GetTrainedModelsAction.INSTANCE, getModelWithDeploymentId, checkDeploymentIdDoesntAlreadyExist); } else { - checkFullModelDefinitionIsPresent(client, trainedModelConfig, true, request.getTimeout(), modelSizeListener); + checkFullModelDefinitionIsPresent(client, trainedModelConfig, true, remainingTime.get(), modelSizeListener); } }, listener::onFailure); @@ -315,16 +318,16 @@ private void getTrainedModelRequestExecution( } private void waitForDeploymentState( - String deploymentId, - TimeValue timeout, - AllocationStatus.State state, + StartTrainedModelDeploymentAction.Request request, + TimeValue remainingTime, ActionListener listener ) { - DeploymentStartedPredicate predicate = new DeploymentStartedPredicate(deploymentId, state); + var deploymentId = request.getDeploymentId(); + DeploymentStartedPredicate predicate = new DeploymentStartedPredicate(deploymentId, request.getWaitForState()); trainedModelAssignmentService.waitForAssignmentCondition( deploymentId, predicate, - timeout, + remainingTime, new TrainedModelAssignmentService.WaitForAssignmentListener() { @Override public void onResponse(TrainedModelAssignment assignment) { @@ -340,6 +343,18 @@ public void onResponse(TrainedModelAssignment assignment) { public void onFailure(Exception e) { listener.onFailure(e); } + + @Override + public void onTimeout(TimeValue timeout) { + onFailure( + new ElasticsearchStatusException( + "Timed out after [{}] waiting for model deployment to start. " + + "Use the trained model stats API to track the state of the deployment.", + RestStatus.REQUEST_TIMEOUT, + request.getTimeout() // use the full request timeout in the error message + ) + ); + } } ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java index f2c2b6de0e19d..0dda155043556 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportTrainedModelCacheInfoAction.java @@ -94,9 +94,7 @@ public static class NodeModelCacheInfoRequest extends TransportRequest { public NodeModelCacheInfoRequest(StreamInput in) throws IOException { super(in); - if (in.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_REQUESTS)) { - new TrainedModelCacheInfoAction.Request(in); - } + skipLegacyNodesRequestHeader(TransportVersions.DROP_UNUSED_NODES_REQUESTS, in); } @Override @@ -107,9 +105,7 @@ public Task createTask(long id, String type, String action, TaskId parentTaskId, @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - if (out.getTransportVersion().before(TransportVersions.DROP_UNUSED_NODES_REQUESTS)) { - new TrainedModelCacheInfoAction.Request().writeTo(out); - } + sendLegacyNodesRequestHeader(TransportVersions.DROP_UNUSED_NODES_REQUESTS, out); } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java index 13f13a271c452..cb32ca01241a8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java @@ -306,7 +306,8 @@ private void buildInferenceStep(DataFrameAnalyticsTask task, DataFrameAnalyticsC config, extractedFields, task.getStatsHolder().getProgressTracker(), - task.getStatsHolder().getDataCountsTracker() + task.getStatsHolder().getDataCountsTracker(), + threadPool ); InferenceStep inferenceStep = new InferenceStep(client, task, auditor, config, threadPool, inferenceRunner); delegate.onResponse(inferenceStep); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java index 073fb13cbf420..dfcc12d98be41 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunner.java @@ -11,13 +11,13 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.PlainActionFuture; -import org.elasticsearch.action.support.UnsafePlainActionFuture; +import org.elasticsearch.action.support.SubscribableListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.client.internal.OriginSettingClient; import org.elasticsearch.common.settings.Settings; @@ -29,6 +29,7 @@ import org.elasticsearch.search.aggregations.metrics.Max; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.tasks.TaskId; +import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.dataframe.DataFrameAnalyticsConfig; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; @@ -69,6 +70,7 @@ public class InferenceRunner { private final ProgressTracker progressTracker; private final DataCountsTracker dataCountsTracker; private final Function testDocsIteratorFactory; + private final ThreadPool threadPool; private volatile boolean isCancelled; InferenceRunner( @@ -81,7 +83,8 @@ public class InferenceRunner { ExtractedFields extractedFields, ProgressTracker progressTracker, DataCountsTracker dataCountsTracker, - Function testDocsIteratorFactory + Function testDocsIteratorFactory, + ThreadPool threadPool ) { this.settings = Objects.requireNonNull(settings); this.client = Objects.requireNonNull(client); @@ -93,43 +96,49 @@ public class InferenceRunner { this.progressTracker = Objects.requireNonNull(progressTracker); this.dataCountsTracker = Objects.requireNonNull(dataCountsTracker); this.testDocsIteratorFactory = Objects.requireNonNull(testDocsIteratorFactory); + this.threadPool = threadPool; } public void cancel() { isCancelled = true; } - public void run(String modelId) { + public void run(String modelId, ActionListener listener) { if (isCancelled) { + listener.onResponse(null); return; } LOGGER.info("[{}] Started inference on test data against model [{}]", config.getId(), modelId); - try { - PlainActionFuture localModelPlainActionFuture = new UnsafePlainActionFuture<>( - MachineLearning.UTILITY_THREAD_POOL_NAME - ); - modelLoadingService.getModelForInternalInference(modelId, localModelPlainActionFuture); + SubscribableListener.newForked(l -> modelLoadingService.getModelForInternalInference(modelId, l)) + .andThen(threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME), threadPool.getThreadContext(), this::handleLocalModel) + .addListener(listener.delegateResponse((delegate, e) -> delegate.onFailure(handleException(modelId, e)))); + } + + private void handleLocalModel(ActionListener listener, LocalModel localModel) { + try (localModel) { InferenceState inferenceState = restoreInferenceState(); dataCountsTracker.setTestDocsCount(inferenceState.processedTestDocsCount); - TestDocsIterator testDocsIterator = testDocsIteratorFactory.apply(inferenceState.lastIncrementalId); - try (LocalModel localModel = localModelPlainActionFuture.actionGet()) { - LOGGER.debug("Loaded inference model [{}]", localModel); - inferTestDocs(localModel, testDocsIterator, inferenceState.processedTestDocsCount); - } - } catch (Exception e) { - LOGGER.error(() -> format("[%s] Error running inference on model [%s]", config.getId(), modelId), e); - if (e instanceof ElasticsearchException elasticsearchException) { - throw new ElasticsearchStatusException( - "[{}] failed running inference on model [{}]; cause was [{}]", - elasticsearchException.status(), - elasticsearchException.getRootCause(), - config.getId(), - modelId, - elasticsearchException.getRootCause().getMessage() - ); - } - throw ExceptionsHelper.serverError( + var testDocsIterator = testDocsIteratorFactory.apply(inferenceState.lastIncrementalId); + LOGGER.debug("Loaded inference model [{}]", localModel); + inferTestDocs(localModel, testDocsIterator, inferenceState.processedTestDocsCount); + listener.onResponse(null); // void + } + } + + private Exception handleException(String modelId, Exception e) { + LOGGER.error(() -> format("[%s] Error running inference on model [%s]", config.getId(), modelId), e); + if (e instanceof ElasticsearchException elasticsearchException) { + return new ElasticsearchStatusException( + "[{}] failed running inference on model [{}]; cause was [{}]", + elasticsearchException.status(), + elasticsearchException.getRootCause(), + config.getId(), + modelId, + elasticsearchException.getRootCause().getMessage() + ); + } else { + return ExceptionsHelper.serverError( "[{}] failed running inference on model [{}]; cause was [{}]", e, config.getId(), @@ -179,6 +188,11 @@ private InferenceState restoreInferenceState() { } private void inferTestDocs(LocalModel model, TestDocsIterator testDocsIterator, long processedTestDocsCount) { + assert ThreadPool.assertCurrentThreadPool(MachineLearning.UTILITY_THREAD_POOL_NAME) + : format( + "inferTestDocs must execute from [MachineLearning.UTILITY_THREAD_POOL_NAME] but thread is [%s]", + Thread.currentThread().getName() + ); long totalDocCount = 0; long processedDocCount = processedTestDocsCount; @@ -255,7 +269,8 @@ public static InferenceRunner create( DataFrameAnalyticsConfig config, ExtractedFields extractedFields, ProgressTracker progressTracker, - DataCountsTracker dataCountsTracker + DataCountsTracker dataCountsTracker, + ThreadPool threadPool ) { return new InferenceRunner( settings, @@ -272,7 +287,8 @@ public static InferenceRunner create( config, extractedFields, lastIncrementalId - ) + ), + threadPool ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/steps/InferenceStep.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/steps/InferenceStep.java index 37ad1a5cb8f56..482e82f9ec303 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/steps/InferenceStep.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/steps/InferenceStep.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.TransportSearchAction; import org.elasticsearch.client.internal.node.NodeClient; @@ -87,18 +88,15 @@ protected void doExecute(ActionListener listener) { } private void runInference(String modelId, ActionListener listener) { - threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(() -> { - try { - inferenceRunner.run(modelId); - listener.onResponse(new StepResponse(isTaskStopping())); - } catch (Exception e) { + threadPool.executor(MachineLearning.UTILITY_THREAD_POOL_NAME).execute(ActionRunnable.wrap(listener, delegate -> { + inferenceRunner.run(modelId, ActionListener.wrap(aVoid -> delegate.onResponse(new StepResponse(isTaskStopping())), e -> { if (task.isStopping()) { - listener.onResponse(new StepResponse(false)); + delegate.onResponse(new StepResponse(false)); } else { - listener.onFailure(e); + delegate.onFailure(e); } - } - }); + })); + })); } private void searchIfTestDocsExist(ActionListener listener) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessor.java index 603abe6394b93..3939bbef4052a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessor.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.ml.inference.nlp; import org.elasticsearch.inference.InferenceResults; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.NlpConfig; import org.elasticsearch.xpack.core.ml.search.WeightedToken; @@ -72,7 +72,7 @@ static InferenceResults processResult( boolean chunkResults ) { if (chunkResults) { - var chunkedResults = new ArrayList(); + var chunkedResults = new ArrayList(); for (int i = 0; i < pyTorchResult.getInferenceResult()[0].length; i++) { int startOffset = tokenization.getTokenization(i).tokens().get(0).get(0).startOffset(); @@ -82,10 +82,10 @@ static InferenceResults processResult( var weightedTokens = sparseVectorToTokenWeights(pyTorchResult.getInferenceResult()[0][i], tokenization, replacementVocab); weightedTokens.sort((t1, t2) -> Float.compare(t2.weight(), t1.weight())); - chunkedResults.add(new InferenceChunkedTextExpansionResults.ChunkedResult(matchedText, weightedTokens)); + chunkedResults.add(new MlChunkedTextExpansionResults.ChunkedResult(matchedText, weightedTokens)); } - return new InferenceChunkedTextExpansionResults( + return new MlChunkedTextExpansionResults( Optional.ofNullable(resultsField).orElse(DEFAULT_RESULTS_FIELD), chunkedResults, tokenization.anyTruncated() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java index b6b050a10c790..cb02990da74c9 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/cat/RestCatJobsAction.java @@ -213,6 +213,12 @@ protected Table getTableWithHeader(RestRequest request) { .setAliases("mbaf", "modelBucketAllocationFailures") .build() ); + table.addCell( + "model.output_memory_allocator_bytes", + TableColumnAttributeBuilder.builder("how many bytes have been used to output the model documents", false) + .setAliases("momab", "modelOutputMemoryAllocatorBytes") + .build() + ); table.addCell( "model.categorization_status", TableColumnAttributeBuilder.builder("current categorization status", false) @@ -416,6 +422,11 @@ private Table buildTable(RestRequest request, Response jobStats) { table.addCell(modelSizeStats == null ? null : modelSizeStats.getTotalPartitionFieldCount()); table.addCell(modelSizeStats == null ? null : modelSizeStats.getBucketAllocationFailuresCount()); table.addCell(modelSizeStats == null ? null : modelSizeStats.getCategorizationStatus().toString()); + table.addCell( + modelSizeStats == null || modelSizeStats.getOutputMemmoryAllocatorBytes() == null + ? null + : ByteSizeValue.ofBytes(modelSizeStats.getOutputMemmoryAllocatorBytes()) + ); table.addCell(modelSizeStats == null ? null : modelSizeStats.getCategorizedDocCount()); table.addCell(modelSizeStats == null ? null : modelSizeStats.getTotalCategoryCount()); table.addCell(modelSizeStats == null ? null : modelSizeStats.getFrequentCategoryCount()); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java index a7b679717c2a0..40cf7d531d5ee 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentAction.java @@ -7,12 +7,15 @@ package org.elasticsearch.xpack.ml.rest.inference; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.client.internal.node.NodeClient; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.RestApiVersion; import org.elasticsearch.core.TimeValue; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.Scope; import org.elasticsearch.rest.ServerlessScope; import org.elasticsearch.rest.action.RestToXContentListener; @@ -24,6 +27,7 @@ import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Objects; import static org.elasticsearch.rest.RestRequest.Method.POST; import static org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction.Request.CACHE_SIZE; @@ -71,7 +75,8 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient String modelId = restRequest.param(StartTrainedModelDeploymentAction.Request.MODEL_ID.getPreferredName()); String deploymentId = restRequest.param(StartTrainedModelDeploymentAction.Request.DEPLOYMENT_ID.getPreferredName(), modelId); StartTrainedModelDeploymentAction.Request request; - if (restRequest.hasContentOrSourceParam()) { + + if (restRequest.hasContentOrSourceParam()) { // request has body request = StartTrainedModelDeploymentAction.Request.parseRequest( modelId, deploymentId, @@ -79,49 +84,110 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient ); } else { request = new StartTrainedModelDeploymentAction.Request(modelId, deploymentId); - if (restRequest.hasParam(TIMEOUT.getPreferredName())) { - TimeValue openTimeout = restRequest.paramAsTime( - TIMEOUT.getPreferredName(), - StartTrainedModelDeploymentAction.DEFAULT_TIMEOUT - ); - request.setTimeout(openTimeout); - } - request.setWaitForState( - AllocationStatus.State.fromString(restRequest.param(WAIT_FOR.getPreferredName(), AllocationStatus.State.STARTED.toString())) - ); - RestCompatibilityChecker.checkAndSetDeprecatedParam( - NUMBER_OF_ALLOCATIONS.getDeprecatedNames()[0], - NUMBER_OF_ALLOCATIONS.getPreferredName(), - RestApiVersion.V_8, - restRequest, - (r, s) -> r.paramAsInt(s, request.getNumberOfAllocations()), - request::setNumberOfAllocations - ); - RestCompatibilityChecker.checkAndSetDeprecatedParam( - THREADS_PER_ALLOCATION.getDeprecatedNames()[0], - THREADS_PER_ALLOCATION.getPreferredName(), - RestApiVersion.V_8, - restRequest, - (r, s) -> r.paramAsInt(s, request.getThreadsPerAllocation()), - request::setThreadsPerAllocation - ); - request.setQueueCapacity(restRequest.paramAsInt(QUEUE_CAPACITY.getPreferredName(), request.getQueueCapacity())); - if (restRequest.hasParam(CACHE_SIZE.getPreferredName())) { - request.setCacheSize( - ByteSizeValue.parseBytesSizeValue(restRequest.param(CACHE_SIZE.getPreferredName()), CACHE_SIZE.getPreferredName()) - ); - } else if (defaultCacheSize != null) { - request.setCacheSize(defaultCacheSize); - } - request.setQueueCapacity(restRequest.paramAsInt(QUEUE_CAPACITY.getPreferredName(), request.getQueueCapacity())); - request.setPriority( - restRequest.param( - StartTrainedModelDeploymentAction.TaskParams.PRIORITY.getPreferredName(), - request.getPriority().toString() + } + + if (restRequest.hasParam(TIMEOUT.getPreferredName())) { + TimeValue openTimeout = validateParameters( + request.getTimeout(), + restRequest.paramAsTime(TIMEOUT.getPreferredName(), StartTrainedModelDeploymentAction.DEFAULT_TIMEOUT), + StartTrainedModelDeploymentAction.DEFAULT_TIMEOUT + ); // hasParam, so never default + request.setTimeout(openTimeout); + } + + request.setWaitForState( + validateParameters( + request.getWaitForState(), + AllocationStatus.State.fromString( + restRequest.param(WAIT_FOR.getPreferredName(), StartTrainedModelDeploymentAction.DEFAULT_WAITFOR_STATE.toString()) + ), + StartTrainedModelDeploymentAction.DEFAULT_WAITFOR_STATE + ) + ); + + RestCompatibilityChecker.checkAndSetDeprecatedParam( + NUMBER_OF_ALLOCATIONS.getDeprecatedNames()[0], + NUMBER_OF_ALLOCATIONS.getPreferredName(), + RestApiVersion.V_8, + restRequest, + (r, s) -> validateParameters( + request.getNumberOfAllocations(), + r.paramAsInt(s, StartTrainedModelDeploymentAction.DEFAULT_NUM_ALLOCATIONS), + StartTrainedModelDeploymentAction.DEFAULT_NUM_ALLOCATIONS + ), + request::setNumberOfAllocations + ); + RestCompatibilityChecker.checkAndSetDeprecatedParam( + THREADS_PER_ALLOCATION.getDeprecatedNames()[0], + THREADS_PER_ALLOCATION.getPreferredName(), + RestApiVersion.V_8, + restRequest, + (r, s) -> validateParameters( + request.getThreadsPerAllocation(), + r.paramAsInt(s, StartTrainedModelDeploymentAction.DEFAULT_NUM_THREADS), + StartTrainedModelDeploymentAction.DEFAULT_NUM_THREADS + ), + request::setThreadsPerAllocation + ); + request.setQueueCapacity( + validateParameters( + request.getQueueCapacity(), + restRequest.paramAsInt(QUEUE_CAPACITY.getPreferredName(), StartTrainedModelDeploymentAction.DEFAULT_QUEUE_CAPACITY), + StartTrainedModelDeploymentAction.DEFAULT_QUEUE_CAPACITY + ) + ); + + if (restRequest.hasParam(CACHE_SIZE.getPreferredName())) { + request.setCacheSize( + validateParameters( + request.getCacheSize(), + ByteSizeValue.parseBytesSizeValue(restRequest.param(CACHE_SIZE.getPreferredName()), CACHE_SIZE.getPreferredName()), + null ) ); + } else if (defaultCacheSize != null && request.getCacheSize() == null) { + request.setCacheSize(defaultCacheSize); } + request.setPriority( + validateParameters( + request.getPriority().toString(), + restRequest.param(StartTrainedModelDeploymentAction.TaskParams.PRIORITY.getPreferredName()), + StartTrainedModelDeploymentAction.DEFAULT_PRIORITY.toString() + ) + ); + return channel -> client.execute(StartTrainedModelDeploymentAction.INSTANCE, request, new RestToXContentListener<>(channel)); } + + /** + * This function validates that the body and query parameters don't conflict, and returns the value that should be used. + * When using this function, the body parameter should already have been set to the default value in + * {@link StartTrainedModelDeploymentAction}, or, set to a different value from the rest request. + * + * @param paramDefault (from {@link StartTrainedModelDeploymentAction}) + * @return the parameter to use + * @throws ElasticsearchStatusException if the parameters don't match + */ + private static T validateParameters(@Nullable T bodyParam, @Nullable T queryParam, @Nullable T paramDefault) + throws ElasticsearchStatusException { + if (Objects.equals(bodyParam, paramDefault) && queryParam != null) { + // the body param is the same as the default for this value. We cannot tell if this was set intentionally, or if it was just the + // default, thus we will assume it was the default + return queryParam; + } + + if (Objects.equals(bodyParam, queryParam)) { + return bodyParam; + } else if (bodyParam == null) { + return queryParam; + } else if (queryParam == null) { + return bodyParam; + } else { + throw new ElasticsearchStatusException( + "The parameter " + bodyParam + " in the body is different from the parameter " + queryParam + " in the query", + RestStatus.BAD_REQUEST + ); + } + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java index ad6b68e1051ff..c86596f237227 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/inference/InferenceRunnerTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.inference.InferenceResults; import org.elasticsearch.search.DocValueFormat; @@ -61,13 +62,18 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; import java.util.stream.Collectors; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -115,7 +121,7 @@ public void testInferTestDocs() { return null; }).when(modelLoadingService).getModelForInternalInference(anyString(), any()); - createInferenceRunner(extractedFields, testDocsIterator).run("model id"); + run(createInferenceRunner(extractedFields, testDocsIterator)).assertSuccess(); var argumentCaptor = ArgumentCaptor.forClass(BulkRequest.class); @@ -146,8 +152,7 @@ public void testInferTestDocs_GivenCancelWasCalled() { InferenceRunner inferenceRunner = createInferenceRunner(extractedFields, infiniteDocsIterator); inferenceRunner.cancel(); - - inferenceRunner.run("model id"); + run(inferenceRunner).assertSuccess(); Mockito.verifyNoMoreInteractions(localModel, resultsPersisterService); assertThat(progressTracker.getInferenceProgressPercent(), equalTo(0)); @@ -178,7 +183,14 @@ private LocalModel localModelInferences(InferenceResults first, InferenceResults return localModel; } + private InferenceRunner createInferenceRunner(ExtractedFields extractedFields) { + return createInferenceRunner(extractedFields, mock(TestDocsIterator.class)); + } + private InferenceRunner createInferenceRunner(ExtractedFields extractedFields, TestDocsIterator testDocsIterator) { + var threadpool = mock(ThreadPool.class); + when(threadpool.executor(any())).thenReturn(EsExecutors.DIRECT_EXECUTOR_SERVICE); + when(threadpool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); return new InferenceRunner( Settings.EMPTY, client, @@ -189,10 +201,52 @@ private InferenceRunner createInferenceRunner(ExtractedFields extractedFields, T extractedFields, progressTracker, new DataCountsTracker(new DataCounts(config.getId())), - id -> testDocsIterator + id -> testDocsIterator, + threadpool ); } + private TestListener run(InferenceRunner inferenceRunner) { + var listener = new TestListener(); + inferenceRunner.run("id", listener); + return listener; + } + + /** + * When an exception is returned in a chained listener's onFailure call + * Then InferenceRunner should wrap it in an ElasticsearchException + */ + public void testModelLoadingServiceResponseWithAnException() { + var expectedCause = new IllegalArgumentException("this is a test"); + doAnswer(ans -> { + ActionListener responseListener = ans.getArgument(1); + responseListener.onFailure(expectedCause); + return null; + }).when(modelLoadingService).getModelForInternalInference(anyString(), any()); + + var actualException = run(createInferenceRunner(mock(ExtractedFields.class))).assertFailure(); + inferenceRunnerHandledException(actualException, expectedCause); + } + + /** + * When an exception is thrown within InferenceRunner + * Then InferenceRunner should wrap it in an ElasticsearchException + */ + public void testExceptionCallingModelLoadingService() { + var expectedCause = new IllegalArgumentException("this is a test"); + + doThrow(expectedCause).when(modelLoadingService).getModelForInternalInference(anyString(), any()); + + var actualException = run(createInferenceRunner(mock(ExtractedFields.class))).assertFailure(); + inferenceRunnerHandledException(actualException, expectedCause); + } + + private void inferenceRunnerHandledException(Exception actual, Exception expectedCause) { + assertThat(actual, instanceOf(ElasticsearchException.class)); + assertThat(actual.getCause(), is(expectedCause)); + assertThat(actual.getMessage(), equalTo("[test] failed running inference on model [id]; cause was [this is a test]")); + } + private Client mockClient() { var client = mock(Client.class); var threadpool = mock(ThreadPool.class); @@ -246,4 +300,28 @@ public SearchResponse actionGet() { } }; } + + private static class TestListener implements ActionListener { + private final AtomicBoolean success = new AtomicBoolean(false); + private final AtomicReference failure = new AtomicReference<>(); + + @Override + public void onResponse(Void t) { + success.set(true); + } + + @Override + public void onFailure(Exception e) { + failure.set(e); + } + + public void assertSuccess() { + assertTrue(success.get()); + } + + public Exception assertFailure() { + assertNotNull(failure.get()); + return failure.get(); + } + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessorTests.java index add071b0a0de0..9803467644db9 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/inference/nlp/TextExpansionProcessorTests.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.ml.inference.nlp; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.inference.results.InferenceChunkedTextExpansionResults; +import org.elasticsearch.xpack.core.ml.inference.results.MlChunkedTextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.BertTokenization; import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfig; @@ -137,9 +137,9 @@ public void testChunking() { var tokenization = tokenizer.tokenize(input, Tokenization.Truncate.NONE, 0, 0, null); var tokenizationResult = new BertTokenizationResult(TEST_CASED_VOCAB, tokenization, 0); var inferenceResult = TextExpansionProcessor.processResult(tokenizationResult, pytorchResult, Map.of(), "foo", true); - assertThat(inferenceResult, instanceOf(InferenceChunkedTextExpansionResults.class)); + assertThat(inferenceResult, instanceOf(MlChunkedTextExpansionResults.class)); - var chunkedResult = (InferenceChunkedTextExpansionResults) inferenceResult; + var chunkedResult = (MlChunkedTextExpansionResults) inferenceResult; assertThat(chunkedResult.getChunks(), hasSize(2)); assertEquals("Elasticsearch darts champion little red", chunkedResult.getChunks().get(0).matchedText()); assertEquals("is fun car", chunkedResult.getChunks().get(1).matchedText()); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentActionTests.java index 26f877a110dc4..7c1f499640e64 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/rest/inference/RestStartTrainedModelDeploymentActionTests.java @@ -8,14 +8,21 @@ package org.elasticsearch.xpack.ml.rest.inference; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.RestActionTestCase; +import org.elasticsearch.xcontent.XContentFactory; +import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.ml.action.CreateTrainedModelAssignmentAction; import org.elasticsearch.xpack.core.ml.action.StartTrainedModelDeploymentAction; import org.elasticsearch.xpack.core.ml.inference.assignment.TrainedModelAssignmentTests; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -64,6 +71,30 @@ public void testCacheEnabled() { assertThat(executeCalled.get(), equalTo(true)); } + public void testExceptionFromDifferentParamsInQueryAndBody() throws IOException { + SetOnce executeCalled = new SetOnce<>(); + controller().registerHandler(new RestStartTrainedModelDeploymentAction(false)); + verifyingClient.setExecuteVerifier(((actionType, actionRequest) -> { + assertThat(actionRequest, instanceOf(StartTrainedModelDeploymentAction.Request.class)); + executeCalled.set(true); + return createResponse(); + })); + + Map paramsMap = new HashMap<>(1); + paramsMap.put("cache_size", "1mb"); + RestRequest inferenceRequest = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.POST) + .withPath("_ml/trained_models/test_id/deployment/_start") + .withParams(paramsMap) + .withContent( + BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("cache_size", "2mb").endObject()), + XContentType.JSON + ) + .build(); + dispatchRequest(inferenceRequest); + assertThat(executeCalled.get(), equalTo(null)); // the duplicate parameter should cause an exception, but the exception isn't + // visible here, so we just check that the request failed + } + private static CreateTrainedModelAssignmentAction.Response createResponse() { return new CreateTrainedModelAssignmentAction.Response(TrainedModelAssignmentTests.randomInstance()); } diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java index 2c5485b8d467f..c89638045a5a8 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/cluster/ClusterStatsMonitoringDocTests.java @@ -589,6 +589,9 @@ public void testToXContent() throws IOException { }, "dense_vector": { "value_count": 0 + }, + "sparse_vector": { + "value_count": 0 } }, "nodes": { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java index c6f02637a8bde..0d1a0374d4fc3 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/collector/indices/IndicesStatsMonitoringDocTests.java @@ -180,7 +180,7 @@ public void testToXContent() throws IOException { private CommonStats mockCommonStats() { final CommonStats commonStats = new CommonStats(CommonStatsFlags.ALL); - commonStats.getDocs().add(new DocsStats(1L, 0L, randomNonNegativeLong())); + commonStats.getDocs().add(new DocsStats(1L, 0L, randomNonNegativeLong() >> 8)); // >> 8 to avoid overflow - we add these things up commonStats.getStore().add(new StoreStats(2L, 0L, 0L)); final IndexingStats.Stats indexingStats = new IndexingStats.Stats(3L, 4L, 0L, 0L, 0L, 0L, 0L, 0L, true, 5L, 0, 0); diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveRequest.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveRequest.java index 752fadf11d58a..a08852f60736f 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveRequest.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/ClearRepositoriesMeteringArchiveRequest.java @@ -7,9 +7,7 @@ package org.elasticsearch.xpack.repositories.metering.action; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; -import org.elasticsearch.common.io.stream.StreamOutput; public final class ClearRepositoriesMeteringArchiveRequest extends BaseNodesRequest { private final long maxVersionToClear; @@ -19,11 +17,6 @@ public ClearRepositoriesMeteringArchiveRequest(long maxVersionToClear, String... this.maxVersionToClear = maxVersionToClear; } - @Override - public void writeTo(StreamOutput out) { - TransportAction.localOnly(); - } - public long getMaxVersionToClear() { return maxVersionToClear; } diff --git a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringRequest.java b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringRequest.java index 95c30d3833aa9..d311273dad76e 100644 --- a/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringRequest.java +++ b/x-pack/plugin/repositories-metering-api/src/main/java/org/elasticsearch/xpack/repositories/metering/action/RepositoriesMeteringRequest.java @@ -7,17 +7,10 @@ package org.elasticsearch.xpack.repositories.metering.action; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; -import org.elasticsearch.common.io.stream.StreamOutput; public final class RepositoriesMeteringRequest extends BaseNodesRequest { public RepositoriesMeteringRequest(String... nodesIds) { super(nodesIds); } - - @Override - public void writeTo(StreamOutput out) { - TransportAction.localOnly(); - } } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 1868b53bfd7e9..18ebe65d87986 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -248,7 +248,7 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng public static final String DATA_TIERS_CACHE_INDEX_PREFERENCE = String.join(",", DataTier.DATA_CONTENT, DataTier.DATA_HOT); private static final int SEARCHABLE_SNAPSHOTS_INDEX_MAPPINGS_VERSION = 1; - private volatile Supplier repositoriesServiceSupplier; + private final SetOnce repositoriesService = new SetOnce<>(); private final SetOnce blobStoreCacheService = new SetOnce<>(); private final SetOnce cacheService = new SetOnce<>(); private final SetOnce> frozenCacheService = new SetOnce<>(); @@ -321,7 +321,7 @@ public Collection createComponents(PluginServices services) { NodeEnvironment nodeEnvironment = services.nodeEnvironment(); final List components = new ArrayList<>(); - this.repositoriesServiceSupplier = services.repositoriesServiceSupplier(); + this.repositoriesService.set(services.repositoriesService()); this.threadPool.set(threadPool); this.failShardsListener.set(new FailShardsOnInvalidLicenseClusterListener(getLicenseState(), services.rerouteService())); if (DiscoveryNode.canContainData(settings)) { @@ -417,7 +417,7 @@ public String getFeatureDescription() { @Override public Map getDirectoryFactories() { return Map.of(SEARCHABLE_SNAPSHOT_STORE_TYPE, (indexSettings, shardPath) -> { - final RepositoriesService repositories = repositoriesServiceSupplier.get(); + final RepositoriesService repositories = repositoriesService.get(); assert repositories != null; final CacheService cache = cacheService.get(); assert cache != null; diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java index c0cec06fd6cf7..5e9ef177bdc6f 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotCacheStoresAction.java @@ -104,11 +104,6 @@ public Request(SnapshotId snapshotId, ShardId shardId, DiscoveryNode[] nodes) { this.snapshotId = snapshotId; this.shardId = shardId; } - - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } } public static final class NodeRequest extends TransportRequest { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java index 9a40b39083139..78d520e984bcf 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/action/cache/TransportSearchableSnapshotsNodeCachesStatsAction.java @@ -97,7 +97,7 @@ protected NodeCachesStatsResponse newNodeResponse(StreamInput in, DiscoveryNode } @Override - protected void resolveRequest(NodesRequest request, ClusterState clusterState) { + protected DiscoveryNode[] resolveRequest(NodesRequest request, ClusterState clusterState) { final Map dataNodes = clusterState.getNodes().getDataNodes(); final DiscoveryNode[] resolvedNodes; @@ -109,7 +109,7 @@ protected void resolveRequest(NodesRequest request, ClusterState clusterState) { .map(dataNodes::get) .toArray(DiscoveryNode[]::new); } - request.setConcreteNodes(resolvedNodes); + return resolvedNodes; } @Override @@ -149,15 +149,9 @@ public void writeTo(StreamOutput out) throws IOException { } public static final class NodesRequest extends BaseNodesRequest { - public NodesRequest(String[] nodes) { super(nodes); } - - @Override - public void writeTo(StreamOutput out) { - TransportAction.localOnly(); - } } public static class NodeCachesStatsResponse extends BaseNodeResponse implements ToXContentFragment { diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index b1989a0db32b9..3dd8d780d6f82 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -153,6 +153,7 @@ public class Constants { "cluster:admin/xpack/connector/secret/put", "indices:data/write/xpack/connector/sync_job/cancel", "indices:data/write/xpack/connector/sync_job/check_in", + "indices:data/write/xpack/connector/sync_job/claim", "indices:data/write/xpack/connector/sync_job/delete", "indices:data/read/xpack/connector/sync_job/get", "indices:data/read/xpack/connector/sync_job/list", @@ -614,6 +615,8 @@ public class Constants { "internal:cluster/formation/info", "internal:gateway/local/started_shards", "internal:admin/indices/prevalidate_shard_path", - "internal:index/metadata/migration_version/update" + "internal:index/metadata/migration_version/update", + "internal:admin/repository/verify", + "internal:admin/repository/verify/coordinate" ).filter(Objects::nonNull).collect(Collectors.toUnmodifiableSet()); } diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ApiKeyAggsIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ApiKeyAggsIT.java index f9d5c42affcf0..d7f19895e1184 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ApiKeyAggsIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/ApiKeyAggsIT.java @@ -246,10 +246,7 @@ public void testFiltersAggs() throws IOException { """); ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(exception.getResponse().toString(), exception.getResponse().getStatusLine().getStatusCode(), is(400)); - assertThat( - exception.getMessage(), - containsString("Field [api_key_invalidated] is not allowed for API Key query or aggregation") - ); + assertThat(exception.getMessage(), containsString("Field [api_key_invalidated] is not allowed for querying or aggregation")); } { Request request = new Request("GET", "/_security/_query/api_key" + (randomBoolean() ? "?typed_keys" : "")); @@ -282,7 +279,7 @@ public void testFiltersAggs() throws IOException { """); ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(exception.getResponse().toString(), exception.getResponse().getStatusLine().getStatusCode(), is(400)); - assertThat(exception.getMessage(), containsString("Field [creator.realm] is not allowed for API Key query or aggregation")); + assertThat(exception.getMessage(), containsString("Field [creator.realm] is not allowed for querying or aggregation")); } } @@ -418,7 +415,7 @@ public void testAggsForType() throws IOException { """); ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(exception.getResponse().toString(), exception.getResponse().getStatusLine().getStatusCode(), is(400)); - assertThat(exception.getMessage(), containsString("Field [runtime_key_type] is not allowed for API Key query or aggregation")); + assertThat(exception.getMessage(), containsString("Field [runtime_key_type] is not allowed for querying or aggregation")); } } @@ -549,7 +546,7 @@ public void testFilterAggs() throws IOException { """); ResponseException exception = expectThrows(ResponseException.class, () -> client().performRequest(request)); assertThat(exception.getResponse().toString(), exception.getResponse().getStatusLine().getStatusCode(), is(400)); - assertThat(exception.getMessage(), containsString("Field [creator] is not allowed for API Key query or aggregation")); + assertThat(exception.getMessage(), containsString("Field [creator] is not allowed for querying or aggregation")); } } diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java index 998343f87ce13..a851b10e6c545 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryApiKeyIT.java @@ -138,7 +138,7 @@ public void testQuery() throws IOException { // Search for fields outside of the allowlist fails ResponseException responseException = assertQueryError(API_KEY_ADMIN_AUTH_HEADER, 400, """ { "query": { "prefix": {"api_key_hash": "{PBKDF2}10000$"} } }"""); - assertThat(responseException.getMessage(), containsString("Field [api_key_hash] is not allowed for API Key query")); + assertThat(responseException.getMessage(), containsString("Field [api_key_hash] is not allowed for querying")); // Search for fields that are not allowed in Query DSL but used internally by the service itself final String fieldName = randomFrom("doc_type", "api_key_invalidated", "invalidation_time"); diff --git a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryUserIT.java b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryUserIT.java index 0d217d201731c..223c07a1e9dec 100644 --- a/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryUserIT.java +++ b/x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/QueryUserIT.java @@ -195,13 +195,13 @@ public void testQuery() throws IOException { assertQueryError(TEST_USER_NO_READ_USERS_AUTH_HEADER, 403, """ { "query": { "wildcard": {"name": "*prefix*"} } }"""); - // Range query not supported + // Span term query not supported assertQueryError(400, """ - {"query":{"range":{"username":{"lt":"now"}}}}"""); + {"query":{"span_term":{"username": "X"} } }"""); - // IDs query not supported + // Fuzzy query not supported assertQueryError(400, """ - { "query": { "ids": { "values": "abc" } } }"""); + { "query": { "fuzzy": { "username": "X" } } }"""); // Make sure we can't query reserved users String reservedUsername = getReservedUsernameAndAssertExists(); @@ -323,8 +323,8 @@ public void testSort() throws IOException { assertQueryError( READ_USERS_USER_AUTH_HEADER, 400, - String.format("{\"sort\":[\"%s\"]}", invalidSortName), - String.format("sorting is not supported for field [%s] in User query", invalidSortName) + Strings.format("{\"sort\":[\"%s\"]}", invalidSortName), + Strings.format("sorting is not supported for field [%s]", invalidSortName) ); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java index bddc765b12d2f..695ea611e599e 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/SecuritySearchOperationListenerTests.java @@ -21,8 +21,8 @@ import org.elasticsearch.search.internal.ShardSearchContextId; import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequest.Empty; import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef; @@ -134,7 +134,7 @@ public void testValidateSearchContext() throws Exception { .realmRef(new RealmRef("realm", "file", "node")) .build(false); authentication.writeToContext(threadContext); - listener.validateReaderContext(readerContext, Empty.INSTANCE); + listener.validateReaderContext(readerContext, new EmptyRequest()); assertThat(threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), is(indicesAccessControl)); verifyNoMoreInteractions(auditTrail); } @@ -147,7 +147,7 @@ public void testValidateSearchContext() throws Exception { .realmRef(new RealmRef(realmName, "file", nodeName)) .build(false); authentication.writeToContext(threadContext); - listener.validateReaderContext(readerContext, Empty.INSTANCE); + listener.validateReaderContext(readerContext, new EmptyRequest()); assertThat(threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY), is(indicesAccessControl)); verifyNoMoreInteractions(auditTrail); } diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreCacheTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreCacheTests.java index d11ca70744b7b..3094a10b1572d 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreCacheTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authz/store/NativePrivilegeStoreCacheTests.java @@ -116,6 +116,7 @@ public void configureApplicationPrivileges() { assertEquals(6, putPrivilegesResponse.created().values().stream().mapToInt(List::size).sum()); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/109894") public void testGetPrivilegesUsesCache() { final Client client = client(); @@ -204,6 +205,7 @@ public void testPopulationOfCacheWhenLoadingPrivilegesForAllApplications() { assertEquals(1, new GetPrivilegesRequestBuilder(client).application("app-1").privileges("write").get().privileges().length); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/109895") public void testSuffixWildcard() { final Client client = client(); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PrivilegedFileWatcher.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PrivilegedFileWatcher.java new file mode 100644 index 0000000000000..583bb93c2a52b --- /dev/null +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/PrivilegedFileWatcher.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.security; + +import org.elasticsearch.watcher.FileWatcher; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.security.PrivilegedAction; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; + +import static java.security.AccessController.doPrivileged; + +/** + * Extension of {@code FileWatcher} that does privileged calls to IO. + *

    + * This class exists so that the calls into the IO methods get here first in the security stackwalk, + * enabling us to use doPrivileged to ensure we have access. If we don't do this, the code location + * that is doing the accessing is not the one that is granted the SecuredFileAccessPermission, + * so the check in ESPolicy fails. + */ +public class PrivilegedFileWatcher extends FileWatcher { + + public PrivilegedFileWatcher(Path path) { + super(path); + } + + @Override + protected boolean fileExists(Path path) { + return doPrivileged((PrivilegedAction) () -> Files.exists(path)); + } + + @Override + protected BasicFileAttributes readAttributes(Path path) throws IOException { + try { + return doPrivileged( + (PrivilegedExceptionAction) () -> Files.readAttributes(path, BasicFileAttributes.class) + ); + } catch (PrivilegedActionException e) { + throw new IOException(e); + } + } + + @Override + protected DirectoryStream listFiles(Path path) throws IOException { + try { + return doPrivileged((PrivilegedExceptionAction>) () -> Files.newDirectoryStream(path)); + } catch (PrivilegedActionException e) { + throw new IOException(e); + } + } +} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 283e8a03c1a95..404b9b85e2b24 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -38,6 +38,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.logging.DeprecationCategory; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.ClusterSettings; @@ -412,6 +414,9 @@ import java.io.Closeable; import java.io.IOException; import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.PrivilegedAction; import java.security.Provider; import java.time.Clock; import java.util.ArrayList; @@ -436,6 +441,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static java.security.AccessController.doPrivileged; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; @@ -697,6 +703,29 @@ protected List getReloadableSecurityComponents() { return this.reloadableComponents.get(); } + /* + * Copied from XPackPlugin.resolveConfigFile so we don't go to a different codesource + * and so fail the secured file permission check on the users file. + * If there's a secured permission granted on this file (which there should be), + * ES has already checked the file is actually in the config directory + */ + public static Path resolveSecuredConfigFile(Environment env, String file) { + Path config = env.configFile().resolve(file); + if (doPrivileged((PrivilegedAction) () -> Files.exists(config)) == false) { + Path legacyConfig = env.configFile().resolve("x-pack").resolve(file); + if (doPrivileged((PrivilegedAction) () -> Files.exists(legacyConfig))) { + DeprecationLogger.getLogger(XPackPlugin.class) + .warn( + DeprecationCategory.OTHER, + "config_file_path", + "Config file [" + file + "] is in a deprecated location. Move from " + legacyConfig + " to " + config + ); + return legacyConfig; + } + } + return config; + } + @Override public Collection createComponents(PluginServices services) { try { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java index 8abc307ab982d..1454b9e480a39 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyAction.java @@ -28,7 +28,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; -import static org.elasticsearch.xpack.security.support.ApiKeyFieldNameTranslators.translateFieldSortBuilders; +import static org.elasticsearch.xpack.security.support.FieldNameTranslators.API_KEY_FIELD_NAME_TRANSLATORS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; public final class TransportQueryApiKeyAction extends TransportAction { @@ -94,7 +94,7 @@ protected void doExecute(Task task, QueryApiKeyRequest request, ActionListener { + API_KEY_FIELD_NAME_TRANSLATORS.translateFieldSortBuilders(request.getFieldSortBuilders(), searchSourceBuilder, fieldName -> { if (API_KEY_TYPE_RUNTIME_MAPPING_FIELD.equals(fieldName)) { accessesApiKeyTypeField.set(true); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserAction.java index ca5b9fc54db47..72f89209b0b79 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserAction.java @@ -16,7 +16,6 @@ import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.tasks.Task; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.security.action.ActionTypes; @@ -30,20 +29,17 @@ import org.elasticsearch.xpack.security.support.UserBoolQueryBuilder; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Set; import java.util.stream.Collectors; +import static org.elasticsearch.xpack.security.support.FieldNameTranslators.USER_FIELD_NAME_TRANSLATORS; import static org.elasticsearch.xpack.security.support.SecuritySystemIndices.SECURITY_MAIN_ALIAS; -import static org.elasticsearch.xpack.security.support.UserBoolQueryBuilder.USER_FIELD_NAME_TRANSLATOR; public final class TransportQueryUserAction extends TransportAction { private final NativeUsersStore usersStore; private final ProfileService profileService; private final Authentication.RealmRef nativeRealmRef; - private static final Set FIELD_NAMES_WITH_SORT_SUPPORT = Set.of("username", "roles", "enabled"); @Inject public TransportQueryUserAction( @@ -76,7 +72,7 @@ protected void doExecute(Task task, QueryUserRequest request, ActionListener fieldSortBuilders, SearchSourceBuilder searchSourceBuilder) { - fieldSortBuilders.forEach(fieldSortBuilder -> { - if (fieldSortBuilder.getNestedSort() != null) { - throw new IllegalArgumentException("nested sorting is not supported for User query"); - } - if (FieldSortBuilder.DOC_FIELD_NAME.equals(fieldSortBuilder.getFieldName())) { - searchSourceBuilder.sort(fieldSortBuilder); - } else { - final String translatedFieldName = USER_FIELD_NAME_TRANSLATOR.translate(fieldSortBuilder.getFieldName()); - if (FIELD_NAMES_WITH_SORT_SUPPORT.contains(translatedFieldName) == false) { - throw new IllegalArgumentException( - String.format(Locale.ROOT, "sorting is not supported for field [%s] in User query", fieldSortBuilder.getFieldName()) - ); - } - - if (translatedFieldName.equals(fieldSortBuilder.getFieldName())) { - searchSourceBuilder.sort(fieldSortBuilder); - } else { - final FieldSortBuilder translatedFieldSortBuilder = new FieldSortBuilder(translatedFieldName).order( - fieldSortBuilder.order() - ) - .missing(fieldSortBuilder.missing()) - .unmappedType(fieldSortBuilder.unmappedType()) - .setFormat(fieldSortBuilder.getFormat()); - - if (fieldSortBuilder.sortMode() != null) { - translatedFieldSortBuilder.sortMode(fieldSortBuilder.sortMode()); - } - if (fieldSortBuilder.getNumericType() != null) { - translatedFieldSortBuilder.setNumericType(fieldSortBuilder.getNumericType()); - } - searchSourceBuilder.sort(translatedFieldSortBuilder); - } - } - }); - } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java index 233f5791a6452..aaa1841bd2354 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java @@ -343,9 +343,21 @@ public void createApiKey( ActionListener listener ) { assert request.getType() != ApiKey.Type.CROSS_CLUSTER || false == authentication.isApiKey() - : "cannot create derived cross-cluster API keys"; + : "cannot create derived cross-cluster API keys (name=[" + + request.getName() + + "], type=[" + + request.getType() + + "], auth=[" + + authentication + + "])"; assert request.getType() != ApiKey.Type.CROSS_CLUSTER || userRoleDescriptors.isEmpty() - : "owner user role descriptor must be empty for cross-cluster API keys"; + : "owner user role descriptor must be empty for cross-cluster API keys (name=[" + + request.getName() + + "], type=[" + + request.getType() + + "], roles=[" + + userRoleDescriptors + + "])"; ensureEnabled(); if (authentication == null) { listener.onFailure(new IllegalArgumentException("authentication must be provided")); @@ -492,7 +504,8 @@ private void createApiKeyAndIndexIt( final Instant created = clock.instant(); final Instant expiration = getApiKeyExpiration(created, request.getExpiration()); final SecureString apiKey = UUIDs.randomBase64UUIDSecureString(); - assert ApiKey.Type.CROSS_CLUSTER != request.getType() || API_KEY_SECRET_LENGTH == apiKey.length(); + assert ApiKey.Type.CROSS_CLUSTER != request.getType() || API_KEY_SECRET_LENGTH == apiKey.length() + : "Invalid API key (name=[" + request.getName() + "], type=[" + request.getType() + "], length=[" + apiKey.length() + "])"; computeHashForApiKey(apiKey, listener.delegateFailure((l, apiKeyHashChars) -> { try ( @@ -528,8 +541,16 @@ private void createApiKeyAndIndexIt( TransportBulkAction.TYPE, bulkRequest, TransportBulkAction.unwrappingSingleItemBulkResponse(ActionListener.wrap(indexResponse -> { - assert request.getId().equals(indexResponse.getId()); - assert indexResponse.getResult() == DocWriteResponse.Result.CREATED; + assert request.getId().equals(indexResponse.getId()) + : "Mismatched API key (request=[" + + request.getId() + + "](name=[" + + request.getName() + + "]) index=[" + + indexResponse.getId() + + "])"; + assert indexResponse.getResult() == DocWriteResponse.Result.CREATED + : "Index response was [" + indexResponse.getResult() + "]"; final ListenableFuture listenableFuture = new ListenableFuture<>(); listenableFuture.onResponse(new CachedApiKeyHashResult(true, apiKey)); apiKeyAuthCache.put(request.getId(), listenableFuture); @@ -552,7 +573,15 @@ public void updateApiKeys( final ActionListener listener ) { assert request.getType() != ApiKey.Type.CROSS_CLUSTER || userRoleDescriptors.isEmpty() - : "owner user role descriptor must be empty for cross-cluster API keys"; + : "owner user role descriptor must be empty for cross-cluster API keys (ids=[" + + (request.getIds().size() <= 10 + ? request.getIds() + : (request.getIds().size() + " including " + request.getIds().subList(0, 10))) + + "], type=[" + + request.getType() + + "], roles=[" + + userRoleDescriptors + + "])"; ensureEnabled(); if (authentication == null) { listener.onFailure(new IllegalArgumentException("authentication must be provided")); @@ -628,10 +657,11 @@ private void updateApiKeys( ) { logger.trace("Found [{}] API keys of [{}] requested for update", targetVersionedDocs.size(), request.getIds().size()); assert targetVersionedDocs.size() <= request.getIds().size() - : "more docs were found for update than were requested. found: " + : "more docs were found for update than were requested. found [" + targetVersionedDocs.size() - + " requested: " - + request.getIds().size(); + + "] requested [" + + request.getIds().size() + + "]"; final BulkUpdateApiKeyResponse.Builder responseBuilder = BulkUpdateApiKeyResponse.builder(); final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk(); @@ -682,7 +712,14 @@ void validateForUpdate( final Authentication authentication, final ApiKeyDoc apiKeyDoc ) { - assert authentication.getEffectiveSubject().getUser().principal().equals(apiKeyDoc.creator.get("principal")); + assert authentication.getEffectiveSubject().getUser().principal().equals(apiKeyDoc.creator.get("principal")) + : "Authenticated user should be owner (authentication=[" + + authentication + + "], owner=[" + + apiKeyDoc.creator + + "], id=[" + + apiKeyId + + "])"; if (apiKeyDoc.invalidated) { throw new IllegalArgumentException("cannot update invalidated API key [" + apiKeyId + "]"); @@ -862,7 +899,14 @@ static XContentBuilder maybeBuildUpdatedDocument( final Set userRoleDescriptors, final Clock clock ) throws IOException { - assert currentApiKeyDoc.type == request.getType(); + assert currentApiKeyDoc.type == request.getType() + : "API Key doc does not match request type (key-id=[" + + apiKeyId + + "], doc=[" + + currentApiKeyDoc.type + + "], request=[" + + request.getType() + + "])"; if (isNoop(apiKeyId, currentApiKeyDoc, targetDocVersion, authentication, request, userRoleDescriptors)) { return null; } @@ -888,7 +932,7 @@ static XContentBuilder maybeBuildUpdatedDocument( logger.trace(() -> format("Building API key doc with updated role descriptors [%s]", keyRoles)); addRoleDescriptors(builder, keyRoles); } else { - assert currentApiKeyDoc.roleDescriptorsBytes != null; + assert currentApiKeyDoc.roleDescriptorsBytes != null : "Role descriptors for [" + apiKeyId + "] are null"; builder.rawField("role_descriptors", currentApiKeyDoc.roleDescriptorsBytes.streamInput(), XContentType.JSON); } @@ -899,7 +943,7 @@ static XContentBuilder maybeBuildUpdatedDocument( assert currentApiKeyDoc.metadataFlattened == null || MetadataUtils.containsReservedMetadata( XContentHelper.convertToMap(currentApiKeyDoc.metadataFlattened, false, XContentType.JSON).v2() - ) == false : "API key doc to be updated contains reserved metadata"; + ) == false : "API key doc [" + apiKeyId + "] to be updated contains reserved metadata"; final Map metadata = request.getMetadata(); if (metadata != null) { logger.trace(() -> format("Building API key doc with updated metadata [%s]", metadata)); @@ -999,7 +1043,7 @@ private static boolean isNoop( } } - assert userRoleDescriptors != null; + assert userRoleDescriptors != null : "API Key [" + apiKeyId + "] has null role descriptors"; final List currentLimitedByRoleDescriptors = parseRoleDescriptorsBytes( apiKeyId, apiKeyDoc.limitedByRoleDescriptorsBytes, @@ -1456,8 +1500,15 @@ public void crossClusterApiKeyUsageStats(ActionListener> lis findApiKeys(boolQuery, true, true, this::convertSearchHitToApiKeyInfo, ActionListener.wrap(apiKeyInfos -> { int ccsKeys = 0, ccrKeys = 0, ccsCcrKeys = 0; for (ApiKey apiKeyInfo : apiKeyInfos) { - assert apiKeyInfo.getType() == ApiKey.Type.CROSS_CLUSTER; - assert apiKeyInfo.getRoleDescriptors().size() == 1; + assert apiKeyInfo.getType() == ApiKey.Type.CROSS_CLUSTER + : "Incorrect API Key type for [" + apiKeyInfo + "] should be [" + ApiKey.Type.CROSS_CLUSTER + "]"; + assert apiKeyInfo.getRoleDescriptors().size() == 1 + : "API Key [" + + apiKeyInfo + + "] has [" + + apiKeyInfo.getRoleDescriptors().size() + + "] role descriptors, but should be 1"; + final List clusterPrivileges = Arrays.asList( apiKeyInfo.getRoleDescriptors().iterator().next().getClusterPrivileges() ); @@ -1614,7 +1665,14 @@ private IndexRequest maybeBuildIndexRequest( } final var targetDocVersion = ApiKey.CURRENT_API_KEY_VERSION; final var currentDocVersion = new ApiKey.Version(currentVersionedDoc.doc().version); - assert currentDocVersion.onOrBefore(targetDocVersion) : "current API key doc version must be on or before target version"; + assert currentDocVersion.onOrBefore(targetDocVersion) + : "API key [" + + currentVersionedDoc.id() + + "] has version [" + + currentDocVersion + + " which is greater than current version [" + + ApiKey.CURRENT_API_KEY_VERSION + + "]"; if (logger.isDebugEnabled() && currentDocVersion.before(targetDocVersion)) { logger.debug( "API key update for [{}] will update version from [{}] to [{}]", @@ -1769,7 +1827,7 @@ private void findVersionedApiKeyDocsForSubject( final String[] apiKeyIds, final ActionListener> listener ) { - assert authentication.isApiKey() == false; + assert authentication.isApiKey() == false : "Authentication [" + authentication + "] is an API key, but should not be"; findApiKeysForUserRealmApiKeyIdAndNameCombination( getOwnersRealmNames(authentication), authentication.getEffectiveSubject().getUser().principal(), @@ -1859,12 +1917,28 @@ private void indexInvalidation( apiKeyIdsToInvalidate.add(apiKeyId); } } - assert false == apiKeyIdsToInvalidate.isEmpty() || false == crossClusterApiKeyIdsToSkip.isEmpty(); + + // noinspection ConstantValue + assert false == apiKeyIdsToInvalidate.isEmpty() || false == crossClusterApiKeyIdsToSkip.isEmpty() + : "There are no API keys but that should never happen, original=[" + + (apiKeys.size() > 10 ? ("size=" + apiKeys.size() + " including " + apiKeys.iterator().next()) : apiKeys) + + "], to-invalidate=[" + + apiKeyIdsToInvalidate + + "], to-skip=[" + + crossClusterApiKeyIdsToSkip + + "]"; + if (apiKeyIdsToInvalidate.isEmpty()) { listener.onResponse(new InvalidateApiKeyResponse(Collections.emptyList(), Collections.emptyList(), failedRequestResponses)); return; } - assert bulkRequestBuilder.numberOfActions() > 0; + assert bulkRequestBuilder.numberOfActions() > 0 + : "Bulk request has [" + + bulkRequestBuilder.numberOfActions() + + "] actions, but there are [" + + apiKeyIdsToInvalidate.size() + + "] api keys to invalidate"; + bulkRequestBuilder.setRefreshPolicy(defaultCreateDocRefreshPolicy(settings)); securityIndex.prepareIndexIfNeededThenExecute( ex -> listener.onFailure(traceLog("prepare security index", ex)), @@ -1932,7 +2006,14 @@ private void buildResponseAndClearCache( ); } else { // Since we made an index request against an existing document, we can't get a NOOP or CREATED here - assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED; + assert bulkItemResponse.getResponse().getResult() == DocWriteResponse.Result.UPDATED + : "Bulk Item [" + + bulkItemResponse.getId() + + "] is [" + + bulkItemResponse.getResponse().getResult() + + "] but should be [" + + DocWriteResponse.Result.UPDATED + + "]"; responseBuilder.updated(apiKeyId); } } @@ -2279,7 +2360,9 @@ public static String[] getOwnersRealmNames(final Authentication authentication) // is no owner information to return here if (effectiveSubjectRealm == null) { final var message = - "Cannot determine owner realms without an effective subject realm for non-API key authentication object"; + "Cannot determine owner realms without an effective subject realm for non-API key authentication object [" + + authentication + + "]"; assert false : message; throw new IllegalArgumentException(message); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java index 9cd1963a1dda0..698cda1683a20 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/esnative/NativeUsersStore.java @@ -79,7 +79,7 @@ */ public class NativeUsersStore { - static final String USER_DOC_TYPE = "user"; + public static final String USER_DOC_TYPE = "user"; public static final String RESERVED_USER_TYPE = "reserved-user"; private static final Logger logger = LogManager.getLogger(NativeUsersStore.class); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java index a3f080f3bf124..c96d77b3134bb 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/FileUserPasswdStore.java @@ -16,7 +16,6 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.watcher.FileWatcher; import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.authc.AuthenticationResult; import org.elasticsearch.xpack.core.security.authc.RealmConfig; @@ -25,6 +24,8 @@ import org.elasticsearch.xpack.core.security.support.Validation; import org.elasticsearch.xpack.core.security.support.Validation.Users; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.PrivilegedFileWatcher; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.support.FileReloadListener; import org.elasticsearch.xpack.security.support.SecurityFiles; @@ -32,6 +33,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.security.PrivilegedAction; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -39,6 +41,7 @@ import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; +import static java.security.AccessController.doPrivileged; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.core.Strings.format; @@ -58,9 +61,9 @@ public FileUserPasswdStore(RealmConfig config, ResourceWatcherService watcherSer FileUserPasswdStore(RealmConfig config, ResourceWatcherService watcherService, Runnable listener) { file = resolveFile(config.env()); settings = config.settings(); - users = parseFileLenient(file, logger, settings); + users = doPrivileged((PrivilegedAction>) () -> parseFileLenient(file, logger, settings)); listeners = new CopyOnWriteArrayList<>(Collections.singletonList(listener)); - FileWatcher watcher = new FileWatcher(file.getParent()); + FileWatcher watcher = new PrivilegedFileWatcher(file.getParent()); watcher.addListener(new FileReloadListener(file, this::tryReload)); try { watcherService.add(watcher, ResourceWatcherService.Frequency.HIGH); @@ -93,7 +96,7 @@ public boolean userExists(String username) { } public static Path resolveFile(Environment env) { - return XPackPlugin.resolveConfigFile(env, "users"); + return Security.resolveSecuredConfigFile(env, "users"); } /** @@ -179,7 +182,7 @@ void notifyRefresh() { private void tryReload() { final Map previousUsers = users; - users = parseFileLenient(file, logger, settings); + users = doPrivileged((PrivilegedAction>) () -> parseFileLenient(file, logger, settings)); if (Maps.deepEquals(previousUsers, users) == false) { logger.info("users file [{}] changed. updating users...", file.toAbsolutePath()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java index eb8b2eb943c7f..c9652652b2d1f 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/file/tool/UsersTool.java @@ -134,7 +134,13 @@ public void execute(Terminal terminal, OptionSet options, Environment env, Proce FileUserPasswdStore.writeFile(users, passwordFile); if (roles.length > 0) { - Map userRoles = new HashMap<>(FileUserRolesStore.parseFile(rolesFile, null)); + final Map userRoles; + if (Files.exists(rolesFile)) { + userRoles = new HashMap<>(FileUserRolesStore.parseFile(rolesFile, null)); + } else { + terminal.println("Roles file [" + rolesFile + "] does not exist, will attempt to create it"); + userRoles = new HashMap<>(); + } userRoles.put(username, roles); FileUserRolesStore.writeFile(userRoles, rolesFile); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java index 9c3714124f4f8..0f8539e69bb32 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java @@ -19,14 +19,16 @@ import org.elasticsearch.watcher.FileChangesListener; import org.elasticsearch.watcher.FileWatcher; import org.elasticsearch.watcher.ResourceWatcherService; -import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.security.authc.RealmConfig; import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings; +import org.elasticsearch.xpack.security.PrivilegedFileWatcher; +import org.elasticsearch.xpack.security.Security; import org.elasticsearch.xpack.security.authc.support.mapper.AbstractRoleMapperClearRealmCache; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.security.PrivilegedAction; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -36,6 +38,7 @@ import java.util.Set; import java.util.stream.Collectors; +import static java.security.AccessController.doPrivileged; import static java.util.Collections.emptyMap; import static java.util.Collections.unmodifiableMap; import static org.elasticsearch.core.Strings.format; @@ -58,8 +61,10 @@ public DnRoleMapper(RealmConfig config, ResourceWatcherService watcherService) { this.config = config; useUnmappedGroupsAsRoles = config.getSetting(DnRoleMapperSettings.USE_UNMAPPED_GROUPS_AS_ROLES_SETTING); file = resolveFile(config); - dnRoles = parseFileLenient(file, logger, config.type(), config.name()); - FileWatcher watcher = new FileWatcher(file.getParent()); + dnRoles = doPrivileged( + (PrivilegedAction>>) () -> parseFileLenient(file, logger, config.type(), config.name()) + ); + FileWatcher watcher = new PrivilegedFileWatcher(file.getParent()); watcher.addListener(new FileListener()); try { watcherService.add(watcher, ResourceWatcherService.Frequency.HIGH); @@ -70,7 +75,7 @@ public DnRoleMapper(RealmConfig config, ResourceWatcherService watcherService) { public static Path resolveFile(RealmConfig realmConfig) { String location = realmConfig.getSetting(DnRoleMapperSettings.ROLE_MAPPING_FILE_SETTING); - return XPackPlugin.resolveConfigFile(realmConfig.env(), location); + return Security.resolveSecuredConfigFile(realmConfig.env(), location); } /** @@ -233,7 +238,9 @@ public void onFileDeleted(Path file) { public void onFileChanged(Path file) { if (file.equals(DnRoleMapper.this.file)) { final Map> previousDnRoles = dnRoles; - dnRoles = parseFileLenient(file, logger, config.type(), config.name()); + dnRoles = doPrivileged( + (PrivilegedAction>>) () -> parseFileLenient(file, logger, config.type(), config.name()) + ); if (previousDnRoles.equals(dnRoles) == false) { logger.info( diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheck.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheck.java index d70552f016bbf..598daea9c2520 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheck.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/RoleMappingFileBootstrapCheck.java @@ -14,6 +14,8 @@ import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings; import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; /** * A BootstrapCheck that {@link DnRoleMapper} files exist and are valid (valid YAML and valid DNs) @@ -31,7 +33,15 @@ public class RoleMappingFileBootstrapCheck implements BootstrapCheck { @Override public BootstrapCheckResult check(BootstrapContext context) { try { - DnRoleMapper.parseFile(path, LogManager.getLogger(getClass()), realmConfig.type(), realmConfig.name(), true); + AccessController.doPrivileged( + (PrivilegedAction) () -> DnRoleMapper.parseFile( + path, + LogManager.getLogger(getClass()), + realmConfig.type(), + realmConfig.name(), + true + ) + ); return BootstrapCheckResult.success(); } catch (Exception e) { return BootstrapCheckResult.failure(e.getMessage()); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java index ff973ce4319f6..4e2e17af2d6f1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/enrollment/InternalEnrollmentTokenGenerator.java @@ -64,8 +64,10 @@ public InternalEnrollmentTokenGenerator(Environment environment, SSLService sslS */ public void maybeCreateNodeEnrollmentToken(Consumer consumer, Iterator backoff) { // the enrollment token can only be used against the node that generated it - final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().nodesIds("_local") - .addMetrics(NodesInfoMetrics.Metric.HTTP.metricName(), NodesInfoMetrics.Metric.TRANSPORT.metricName()); + final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest("_local").addMetrics( + NodesInfoMetrics.Metric.HTTP.metricName(), + NodesInfoMetrics.Metric.TRANSPORT.metricName() + ); client.execute(TransportNodesInfoAction.TYPE, nodesInfoRequest, ActionListener.wrap(response -> { assert response.getNodes().size() == 1; @@ -132,8 +134,7 @@ public void maybeCreateNodeEnrollmentToken(Consumer consumer, Iterator consumer, Iterator backoff) { // the enrollment token can only be used against the node that generated it - final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest().nodesIds("_local") - .addMetric(NodesInfoMetrics.Metric.HTTP.metricName()); + final NodesInfoRequest nodesInfoRequest = new NodesInfoRequest("_local").addMetric(NodesInfoMetrics.Metric.HTTP.metricName()); client.execute(TransportNodesInfoAction.TYPE, nodesInfoRequest, ActionListener.wrap(response -> { assert response.getNodes().size() == 1; NodeInfo nodeInfo = response.getNodes().get(0); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyAction.java index 59992e42d88d5..6f62b87ea715e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/apikey/RestQueryApiKeyAction.java @@ -113,6 +113,7 @@ public String getName() { @Override protected Set responseParams() { + // this is a parameter that's consumed by the response formatter for aggregations return Set.of(RestSearchAction.TYPED_KEYS_PARAM); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/NativeRoleBaseRestHandler.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/NativeRoleBaseRestHandler.java index 773d0a8a5ecfd..d19e59c2d6178 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/NativeRoleBaseRestHandler.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/role/NativeRoleBaseRestHandler.java @@ -43,6 +43,5 @@ protected Exception innerCheckFeatureAvailable(RestRequest request) { } else { return null; } - } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java index 5c9d68d3c8b66..a896d4855b73d 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/action/user/RestHasPrivilegesAction.java @@ -91,8 +91,7 @@ public RestChannelConsumer innerPrepareRequest(RestRequest request, NodeClient c if (username == null) { return restChannel -> { throw new ElasticsearchSecurityException("there is no authenticated user"); }; } - HasPrivilegesRequestBuilder requestBuilder = builderFactory.create(client, request.hasParam(RestRequest.PATH_RESTRICTED)) - .source(username, content.v2(), content.v1()); + final HasPrivilegesRequestBuilder requestBuilder = builderFactory.create(client).source(username, content.v2(), content.v1()); return channel -> requestBuilder.execute(new RestBuilderListener<>(channel) { @Override public RestResponse buildResponse(HasPrivilegesResponse response, XContentBuilder builder) throws Exception { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyAggregationsBuilder.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyAggregationsBuilder.java index 495ad1591b6da..3ada85c2129e4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyAggregationsBuilder.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyAggregationsBuilder.java @@ -27,7 +27,7 @@ import java.util.function.Consumer; -import static org.elasticsearch.xpack.security.support.ApiKeyFieldNameTranslators.translateQueryBuilderFields; +import static org.elasticsearch.xpack.security.support.FieldNameTranslators.API_KEY_FIELD_NAME_TRANSLATORS; public class ApiKeyAggregationsBuilder { @@ -73,7 +73,7 @@ private static AggregationBuilder translateAggsFields(AggregationBuilder aggsBui throw new IllegalArgumentException("Unsupported script value source for [" + copiedAggsBuilder.getName() + "] agg"); } // the user-facing field names are different from the index mapping field names of API Key docs - String translatedFieldName = ApiKeyFieldNameTranslators.translate(valuesSourceAggregationBuilder.field()); + String translatedFieldName = API_KEY_FIELD_NAME_TRANSLATORS.translate(valuesSourceAggregationBuilder.field()); valuesSourceAggregationBuilder.field(translatedFieldName); fieldNameVisitor.accept(translatedFieldName); return valuesSourceAggregationBuilder; @@ -88,7 +88,7 @@ private static AggregationBuilder translateAggsFields(AggregationBuilder aggsBui + "]" ); } - String translatedFieldName = ApiKeyFieldNameTranslators.translate(valueSource.field()); + String translatedFieldName = API_KEY_FIELD_NAME_TRANSLATORS.translate(valueSource.field()); valueSource.field(translatedFieldName); fieldNameVisitor.accept(translatedFieldName); } @@ -97,7 +97,7 @@ private static AggregationBuilder translateAggsFields(AggregationBuilder aggsBui // filters the aggregation query to user's allowed API Keys only FilterAggregationBuilder newFilterAggregationBuilder = new FilterAggregationBuilder( filterAggregationBuilder.getName(), - translateQueryBuilderFields(filterAggregationBuilder.getFilter(), fieldNameVisitor) + API_KEY_FIELD_NAME_TRANSLATORS.translateQueryBuilderFields(filterAggregationBuilder.getFilter(), fieldNameVisitor) ); if (filterAggregationBuilder.getMetadata() != null) { newFilterAggregationBuilder.setMetadata(filterAggregationBuilder.getMetadata()); @@ -110,7 +110,7 @@ private static AggregationBuilder translateAggsFields(AggregationBuilder aggsBui // filters the aggregation's bucket queries to user's allowed API Keys only QueryBuilder[] filterQueryBuilders = new QueryBuilder[filtersAggregationBuilder.filters().size()]; for (int i = 0; i < filtersAggregationBuilder.filters().size(); i++) { - filterQueryBuilders[i] = translateQueryBuilderFields( + filterQueryBuilders[i] = API_KEY_FIELD_NAME_TRANSLATORS.translateQueryBuilderFields( filtersAggregationBuilder.filters().get(i).filter(), fieldNameVisitor ); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java index 8d167954b399a..a09d5347e2fe1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilder.java @@ -22,26 +22,11 @@ import java.util.Set; import java.util.function.Consumer; -import static org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD; -import static org.elasticsearch.xpack.security.support.ApiKeyFieldNameTranslators.translateQueryBuilderFields; +import static org.elasticsearch.xpack.security.support.FieldNameTranslators.API_KEY_FIELD_NAME_TRANSLATORS; -public class ApiKeyBoolQueryBuilder extends BoolQueryBuilder { +public final class ApiKeyBoolQueryBuilder extends BoolQueryBuilder { - // Field names allowed at the index level - private static final Set ALLOWED_EXACT_INDEX_FIELD_NAMES = Set.of( - "_id", - "doc_type", - "name", - "type", - API_KEY_TYPE_RUNTIME_MAPPING_FIELD, - "api_key_invalidated", - "invalidation_time", - "creation_time", - "expiration_time", - "metadata_flattened", - "creator.principal", - "creator.realm" - ); + private static final Set FIELDS_ALLOWED_TO_QUERY = Set.of("_id", "doc_type", "type"); private ApiKeyBoolQueryBuilder() {} @@ -69,7 +54,7 @@ public static ApiKeyBoolQueryBuilder build( ) { final ApiKeyBoolQueryBuilder finalQuery = new ApiKeyBoolQueryBuilder(); if (queryBuilder != null) { - QueryBuilder processedQuery = translateQueryBuilderFields(queryBuilder, fieldNameVisitor); + QueryBuilder processedQuery = API_KEY_FIELD_NAME_TRANSLATORS.translateQueryBuilderFields(queryBuilder, fieldNameVisitor); finalQuery.must(processedQuery); } finalQuery.filter(QueryBuilders.termQuery("doc_type", "api_key")); @@ -110,7 +95,6 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws } static boolean isIndexFieldNameAllowed(String fieldName) { - return ALLOWED_EXACT_INDEX_FIELD_NAMES.contains(fieldName) || fieldName.startsWith("metadata_flattened."); + return FIELDS_ALLOWED_TO_QUERY.contains(fieldName) || API_KEY_FIELD_NAME_TRANSLATORS.isIndexFieldSupported(fieldName); } - } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FieldNameTranslators.java similarity index 67% rename from x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java rename to x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FieldNameTranslators.java index f8ea0663a7c51..6d0b076fd9bf1 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/ApiKeyFieldNameTranslators.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/FieldNameTranslators.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.ExistsQueryBuilder; @@ -35,75 +36,42 @@ import java.util.Objects; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Function; import static org.elasticsearch.xpack.security.action.apikey.TransportQueryApiKeyAction.API_KEY_TYPE_RUNTIME_MAPPING_FIELD; -/** - * A class to translate query level field names to index level field names. - */ -public class ApiKeyFieldNameTranslators { - static final List FIELD_NAME_TRANSLATORS; - - static { - FIELD_NAME_TRANSLATORS = List.of( - new ExactFieldNameTranslator(s -> "creator.principal", "username"), - new ExactFieldNameTranslator(s -> "creator.realm", "realm_name"), - new ExactFieldNameTranslator(s -> "name", "name"), - new ExactFieldNameTranslator(s -> API_KEY_TYPE_RUNTIME_MAPPING_FIELD, "type"), - new ExactFieldNameTranslator(s -> "creation_time", "creation"), - new ExactFieldNameTranslator(s -> "expiration_time", "expiration"), - new ExactFieldNameTranslator(s -> "api_key_invalidated", "invalidated"), - new ExactFieldNameTranslator(s -> "invalidation_time", "invalidation"), - // allows querying on all metadata values as keywords because "metadata_flattened" is a flattened field type - new ExactFieldNameTranslator(s -> "metadata_flattened", "metadata"), - new PrefixFieldNameTranslator(s -> "metadata_flattened." + s.substring("metadata.".length()), "metadata.") - ); - } +public final class FieldNameTranslators { - /** - * Adds the {@param fieldSortBuilders} to the {@param searchSourceBuilder}, translating the field names, - * form query level to index level, see {@link #translate}. - * The optional {@param visitor} can be used to collect all the translated field names. - */ - public static void translateFieldSortBuilders( - List fieldSortBuilders, - SearchSourceBuilder searchSourceBuilder, - @Nullable Consumer visitor - ) { - final Consumer fieldNameVisitor = visitor != null ? visitor : ignored -> {}; - fieldSortBuilders.forEach(fieldSortBuilder -> { - if (fieldSortBuilder.getNestedSort() != null) { - throw new IllegalArgumentException("nested sorting is not supported for API Key query"); - } - if (FieldSortBuilder.DOC_FIELD_NAME.equals(fieldSortBuilder.getFieldName())) { - searchSourceBuilder.sort(fieldSortBuilder); - } else { - final String translatedFieldName = translate(fieldSortBuilder.getFieldName()); - fieldNameVisitor.accept(translatedFieldName); - if (translatedFieldName.equals(fieldSortBuilder.getFieldName())) { - searchSourceBuilder.sort(fieldSortBuilder); - } else { - final FieldSortBuilder translatedFieldSortBuilder = new FieldSortBuilder(translatedFieldName).order( - fieldSortBuilder.order() - ) - .missing(fieldSortBuilder.missing()) - .unmappedType(fieldSortBuilder.unmappedType()) - .setFormat(fieldSortBuilder.getFormat()); + public static final FieldNameTranslators API_KEY_FIELD_NAME_TRANSLATORS = new FieldNameTranslators( + List.of( + new SimpleFieldNameTranslator("creator.principal", "username"), + new SimpleFieldNameTranslator("creator.realm", "realm_name"), + new SimpleFieldNameTranslator("name", "name"), + new SimpleFieldNameTranslator(API_KEY_TYPE_RUNTIME_MAPPING_FIELD, "type"), + new SimpleFieldNameTranslator("creation_time", "creation"), + new SimpleFieldNameTranslator("expiration_time", "expiration"), + new SimpleFieldNameTranslator("api_key_invalidated", "invalidated"), + new SimpleFieldNameTranslator("invalidation_time", "invalidation"), + // allows querying on any non-wildcard sub-fields under the "metadata." prefix + // also allows querying on the "metadata" field itself (including by specifying patterns) + new FlattenedFieldNameTranslator("metadata_flattened", "metadata") + ) + ); - if (fieldSortBuilder.sortMode() != null) { - translatedFieldSortBuilder.sortMode(fieldSortBuilder.sortMode()); - } - if (fieldSortBuilder.getNestedSort() != null) { - translatedFieldSortBuilder.setNestedSort(fieldSortBuilder.getNestedSort()); - } - if (fieldSortBuilder.getNumericType() != null) { - translatedFieldSortBuilder.setNumericType(fieldSortBuilder.getNumericType()); - } - searchSourceBuilder.sort(translatedFieldSortBuilder); - } - } - }); + public static final FieldNameTranslators USER_FIELD_NAME_TRANSLATORS = new FieldNameTranslators( + List.of( + idemFieldNameTranslator("username"), + idemFieldNameTranslator("roles"), + idemFieldNameTranslator("enabled"), + // the mapping for these fields does not support sorting (because their mapping does not store "fielddata" in the index) + idemFieldNameTranslator("full_name", false), + idemFieldNameTranslator("email", false) + ) + ); + + private final List fieldNameTranslators; + + private FieldNameTranslators(List fieldNameTranslators) { + this.fieldNameTranslators = fieldNameTranslators; } /** @@ -114,7 +82,7 @@ public static void translateFieldSortBuilders( * associated query level field names match the pattern. * The optional {@param visitor} can be used to collect all the translated field names. */ - public static QueryBuilder translateQueryBuilderFields(QueryBuilder queryBuilder, @Nullable Consumer visitor) { + public QueryBuilder translateQueryBuilderFields(QueryBuilder queryBuilder, @Nullable Consumer visitor) { Objects.requireNonNull(queryBuilder, "unsupported \"null\" query builder for field name translation"); final Consumer fieldNameVisitor = visitor != null ? visitor : ignored -> {}; if (queryBuilder instanceof final BoolQueryBuilder query) { @@ -147,7 +115,7 @@ public static QueryBuilder translateQueryBuilderFields(QueryBuilder queryBuilder return QueryBuilders.existsQuery(translatedFieldName).boost(query.boost()).queryName(query.queryName()); } else if (queryBuilder instanceof final TermsQueryBuilder query) { if (query.termsLookup() != null) { - throw new IllegalArgumentException("terms query with terms lookup is not supported for API Key query"); + throw new IllegalArgumentException("terms query with terms lookup is not currently supported in this context"); } final String translatedFieldName = translate(query.fieldName()); fieldNameVisitor.accept(translatedFieldName); @@ -200,7 +168,7 @@ public static QueryBuilder translateQueryBuilderFields(QueryBuilder queryBuilder return matchQueryBuilder; } else if (queryBuilder instanceof final RangeQueryBuilder query) { if (query.relation() != null) { - throw new IllegalArgumentException("range query with relation is not supported for API Key query"); + throw new IllegalArgumentException("range query with relation is not currently supported in this context"); } final String translatedFieldName = translate(query.fieldName()); fieldNameVisitor.accept(translatedFieldName); @@ -264,35 +232,91 @@ public static QueryBuilder translateQueryBuilderFields(QueryBuilder queryBuilder .boost(query.boost()) .queryName(query.queryName()); } else { - throw new IllegalArgumentException("Query type [" + queryBuilder.getName() + "] is not supported for API Key query"); + throw new IllegalArgumentException("Query type [" + queryBuilder.getName() + "] is not currently supported in this context"); } } + /** + * Adds the {@param fieldSortBuilders} to the {@param searchSourceBuilder}, translating the field names, + * form query level to index level, see {@link #translate}. + * The optional {@param visitor} can be used to collect all the translated field names. + */ + public void translateFieldSortBuilders( + List fieldSortBuilders, + SearchSourceBuilder searchSourceBuilder, + @Nullable Consumer visitor + ) { + final Consumer fieldNameVisitor = visitor != null ? visitor : ignored -> {}; + fieldSortBuilders.forEach(fieldSortBuilder -> { + if (fieldSortBuilder.getNestedSort() != null) { + throw new IllegalArgumentException("nested sorting is not currently supported in this context"); + } + if (FieldSortBuilder.DOC_FIELD_NAME.equals(fieldSortBuilder.getFieldName())) { + searchSourceBuilder.sort(fieldSortBuilder); + } else { + final String translatedFieldName = translate(fieldSortBuilder.getFieldName(), true); + fieldNameVisitor.accept(translatedFieldName); + if (translatedFieldName.equals(fieldSortBuilder.getFieldName())) { + searchSourceBuilder.sort(fieldSortBuilder); + } else { + final FieldSortBuilder translatedFieldSortBuilder = new FieldSortBuilder(translatedFieldName).order( + fieldSortBuilder.order() + ) + .missing(fieldSortBuilder.missing()) + .unmappedType(fieldSortBuilder.unmappedType()) + .setFormat(fieldSortBuilder.getFormat()); + + if (fieldSortBuilder.sortMode() != null) { + translatedFieldSortBuilder.sortMode(fieldSortBuilder.sortMode()); + } + if (fieldSortBuilder.getNestedSort() != null) { + translatedFieldSortBuilder.setNestedSort(fieldSortBuilder.getNestedSort()); + } + if (fieldSortBuilder.getNumericType() != null) { + translatedFieldSortBuilder.setNumericType(fieldSortBuilder.getNumericType()); + } + searchSourceBuilder.sort(translatedFieldSortBuilder); + } + } + }); + } + /** * Translate the query level field name to index level field names. * It throws an exception if the field name is not explicitly allowed. */ - protected static String translate(String fieldName) { + public String translate(String queryFieldName) { + return translate(queryFieldName, false); + } + + /** + * Translate the query level field name to index level field names. + * It throws an exception if the field name is not explicitly allowed. + */ + private String translate(String queryFieldName, boolean inSortContext) { // protected for testing - if (Regex.isSimpleMatchPattern(fieldName)) { - throw new IllegalArgumentException("Field name pattern [" + fieldName + "] is not allowed for API Key query or aggregation"); + if (Regex.isSimpleMatchPattern(queryFieldName)) { + throw new IllegalArgumentException("Field name pattern [" + queryFieldName + "] is not allowed for querying or aggregation"); } - for (FieldNameTranslator translator : FIELD_NAME_TRANSLATORS) { - if (translator.supports(fieldName)) { - return translator.translate(fieldName); + for (FieldNameTranslator translator : fieldNameTranslators) { + if (translator.isQueryFieldSupported(queryFieldName)) { + if (inSortContext && translator.isSortSupported() == false) { + throw new IllegalArgumentException(Strings.format("sorting is not supported for field [%s]", queryFieldName)); + } + return translator.translate(queryFieldName); } } - throw new IllegalArgumentException("Field [" + fieldName + "] is not allowed for API Key query or aggregation"); + throw new IllegalArgumentException("Field [" + queryFieldName + "] is not allowed for querying or aggregation"); } /** * Translates a query level field name pattern to the matching index level field names. * The result can be the empty set, if the pattern doesn't match any of the allowed index level field names. */ - private static Set translatePattern(String fieldNameOrPattern) { + public Set translatePattern(String fieldNameOrPattern) { Set indexFieldNames = new HashSet<>(); - for (FieldNameTranslator translator : FIELD_NAME_TRANSLATORS) { - if (translator.supports(fieldNameOrPattern)) { + for (FieldNameTranslator translator : fieldNameTranslators) { + if (translator.isQueryFieldSupported(fieldNameOrPattern)) { indexFieldNames.add(translator.translate(fieldNameOrPattern)); } } @@ -302,58 +326,112 @@ private static Set translatePattern(String fieldNameOrPattern) { return indexFieldNames; } - abstract static class FieldNameTranslator { + public boolean isQueryFieldSupported(String fieldName) { + return fieldNameTranslators.stream().anyMatch(t -> t.isQueryFieldSupported(fieldName)); + } + + public boolean isIndexFieldSupported(String fieldName) { + return fieldNameTranslators.stream().anyMatch(t -> t.isIndexFieldSupported(fieldName)); + } - private final Function translationFunc; + private interface FieldNameTranslator { + String translate(String fieldName); - protected FieldNameTranslator(Function translationFunc) { - this.translationFunc = translationFunc; - } + boolean isQueryFieldSupported(String fieldName); - String translate(String fieldName) { - return translationFunc.apply(fieldName); - } + boolean isIndexFieldSupported(String fieldName); + + boolean isSortSupported(); + } + + private static SimpleFieldNameTranslator idemFieldNameTranslator(String fieldName) { + return new SimpleFieldNameTranslator(fieldName, fieldName); + } - abstract boolean supports(String fieldName); + private static SimpleFieldNameTranslator idemFieldNameTranslator(String fieldName, boolean isSortSupported) { + return new SimpleFieldNameTranslator(fieldName, fieldName, isSortSupported); } - static class ExactFieldNameTranslator extends FieldNameTranslator { - private final String name; + private static class SimpleFieldNameTranslator implements FieldNameTranslator { + private final String indexFieldName; + private final String queryFieldName; + private final boolean isSortSupported; + + SimpleFieldNameTranslator(String indexFieldName, String queryFieldName, boolean isSortSupported) { + this.indexFieldName = indexFieldName; + this.queryFieldName = queryFieldName; + this.isSortSupported = isSortSupported; + } - ExactFieldNameTranslator(Function translationFunc, String name) { - super(translationFunc); - this.name = name; + SimpleFieldNameTranslator(String indexFieldName, String queryFieldName) { + this(indexFieldName, queryFieldName, true); } @Override - public boolean supports(String fieldNameOrPattern) { + public boolean isQueryFieldSupported(String fieldNameOrPattern) { if (Regex.isSimpleMatchPattern(fieldNameOrPattern)) { - return Regex.simpleMatch(fieldNameOrPattern, name); + return Regex.simpleMatch(fieldNameOrPattern, queryFieldName); } else { - return name.equals(fieldNameOrPattern); + return queryFieldName.equals(fieldNameOrPattern); } } + + @Override + public boolean isIndexFieldSupported(String fieldName) { + return fieldName.equals(indexFieldName); + } + + @Override + public String translate(String fieldNameOrPattern) { + return indexFieldName; + } + + @Override + public boolean isSortSupported() { + return isSortSupported; + } } - static class PrefixFieldNameTranslator extends FieldNameTranslator { - private final String prefix; + private static class FlattenedFieldNameTranslator implements FieldNameTranslator { + private final String indexFieldName; + private final String queryFieldName; - PrefixFieldNameTranslator(Function translationFunc, String prefix) { - super(translationFunc); - this.prefix = prefix; + FlattenedFieldNameTranslator(String indexFieldName, String queryFieldName) { + this.indexFieldName = indexFieldName; + this.queryFieldName = queryFieldName; } @Override - boolean supports(String fieldNamePrefix) { - // a pattern can generally match a prefix in multiple ways - // moreover, it's not possible to iterate the concrete fields matching the prefix - if (Regex.isSimpleMatchPattern(fieldNamePrefix)) { - // this means that e.g. `metadata.*` and `metadata.x*` are expanded to the empty list, - // rather than be replaced with `metadata_flattened.*` and `metadata_flattened.x*` - // (but, in any case, `metadata_flattened.*` and `metadata.x*` are going to be ignored) - return false; + public boolean isQueryFieldSupported(String fieldNameOrPattern) { + if (Regex.isSimpleMatchPattern(fieldNameOrPattern)) { + // It is not possible to translate a pattern for subfields of a flattened field + // (because there's no list of subfields of the flattened field). + // But the pattern can still match the flattened field itself. + return Regex.simpleMatch(fieldNameOrPattern, queryFieldName); + } else { + return fieldNameOrPattern.equals(queryFieldName) || fieldNameOrPattern.startsWith(queryFieldName + "."); } - return fieldNamePrefix.startsWith(prefix); + } + + @Override + public boolean isIndexFieldSupported(String fieldName) { + return fieldName.equals(indexFieldName) || fieldName.startsWith(indexFieldName + "."); + } + + @Override + public String translate(String fieldNameOrPattern) { + if (Regex.isSimpleMatchPattern(fieldNameOrPattern) || fieldNameOrPattern.equals(queryFieldName)) { + // the pattern can only refer to the flattened field itself, not to its subfields + return indexFieldName; + } else { + assert fieldNameOrPattern.startsWith(queryFieldName + "."); + return indexFieldName + fieldNameOrPattern.substring(queryFieldName.length()); + } + } + + @Override + public boolean isSortSupported() { + return true; } } } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexFieldNameTranslator.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexFieldNameTranslator.java deleted file mode 100644 index e262454af2958..0000000000000 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexFieldNameTranslator.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.security.support; - -import java.util.List; -import java.util.function.Function; -import java.util.function.Predicate; - -public class SecurityIndexFieldNameTranslator { - - private final List fieldNameTranslators; - - public SecurityIndexFieldNameTranslator(List fieldNameTranslators) { - this.fieldNameTranslators = fieldNameTranslators; - } - - public String translate(String queryFieldName) { - for (FieldName fieldName : this.fieldNameTranslators) { - if (fieldName.supportsQueryName(queryFieldName)) { - return fieldName.indexFieldName(queryFieldName); - } - } - throw new IllegalArgumentException("Field [" + queryFieldName + "] is not allowed"); - } - - public boolean supportedIndexFieldName(String indexFieldName) { - for (FieldName fieldName : this.fieldNameTranslators) { - if (fieldName.supportsIndexName(indexFieldName)) { - return true; - } - } - return false; - } - - public static FieldName exact(String name) { - return exact(name, Function.identity()); - } - - public static FieldName exact(String name, Function translation) { - return new SecurityIndexFieldNameTranslator.FieldName(name, translation); - } - - public static class FieldName { - private final String name; - private final Function toIndexFieldName; - protected final Predicate validIndexNamePredicate; - - private FieldName(String name, Function toIndexFieldName) { - this.name = name; - this.toIndexFieldName = toIndexFieldName; - this.validIndexNamePredicate = fieldName -> toIndexFieldName.apply(name).equals(fieldName); - - } - - public boolean supportsQueryName(String queryFieldName) { - return queryFieldName.equals(name); - } - - public boolean supportsIndexName(String indexFieldName) { - return validIndexNamePredicate.test(indexFieldName); - } - - public String indexFieldName(String queryFieldName) { - return toIndexFieldName.apply(queryFieldName); - } - } -} diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/UserBoolQueryBuilder.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/UserBoolQueryBuilder.java index 5d3824ab1f8ce..7b476395697ab 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/UserBoolQueryBuilder.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/UserBoolQueryBuilder.java @@ -9,75 +9,33 @@ import org.apache.lucene.search.Query; import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.ExistsQueryBuilder; -import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.PrefixQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.index.query.TermQueryBuilder; -import org.elasticsearch.index.query.TermsQueryBuilder; -import org.elasticsearch.index.query.WildcardQueryBuilder; +import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.authc.esnative.NativeUsersStore; import java.io.IOException; -import java.util.List; +import java.util.Set; -import static org.elasticsearch.xpack.security.support.SecurityIndexFieldNameTranslator.exact; +import static org.elasticsearch.xpack.security.support.FieldNameTranslators.USER_FIELD_NAME_TRANSLATORS; -public class UserBoolQueryBuilder extends BoolQueryBuilder { - public static final SecurityIndexFieldNameTranslator USER_FIELD_NAME_TRANSLATOR = new SecurityIndexFieldNameTranslator( - List.of(exact("username"), exact("roles"), exact("full_name"), exact("email"), exact("enabled")) - ); +public final class UserBoolQueryBuilder extends BoolQueryBuilder { + + private static final Set FIELDS_ALLOWED_TO_QUERY = Set.of("_id", User.Fields.TYPE.getPreferredName()); private UserBoolQueryBuilder() {} public static UserBoolQueryBuilder build(QueryBuilder queryBuilder) { - UserBoolQueryBuilder userQueryBuilder = new UserBoolQueryBuilder(); + final UserBoolQueryBuilder finalQuery = new UserBoolQueryBuilder(); if (queryBuilder != null) { - QueryBuilder translaterdQueryBuilder = translateToUserQueryBuilder(queryBuilder); - userQueryBuilder.must(translaterdQueryBuilder); + QueryBuilder processedQuery = USER_FIELD_NAME_TRANSLATORS.translateQueryBuilderFields(queryBuilder, null); + finalQuery.must(processedQuery); } - userQueryBuilder.filter(QueryBuilders.termQuery("type", "user")); - - return userQueryBuilder; - } + finalQuery.filter(QueryBuilders.termQuery(User.Fields.TYPE.getPreferredName(), NativeUsersStore.USER_DOC_TYPE)); - private static QueryBuilder translateToUserQueryBuilder(QueryBuilder qb) { - if (qb instanceof final BoolQueryBuilder query) { - final BoolQueryBuilder newQuery = QueryBuilders.boolQuery() - .minimumShouldMatch(query.minimumShouldMatch()) - .adjustPureNegative(query.adjustPureNegative()); - query.must().stream().map(UserBoolQueryBuilder::translateToUserQueryBuilder).forEach(newQuery::must); - query.should().stream().map(UserBoolQueryBuilder::translateToUserQueryBuilder).forEach(newQuery::should); - query.mustNot().stream().map(UserBoolQueryBuilder::translateToUserQueryBuilder).forEach(newQuery::mustNot); - query.filter().stream().map(UserBoolQueryBuilder::translateToUserQueryBuilder).forEach(newQuery::filter); - return newQuery; - } else if (qb instanceof MatchAllQueryBuilder) { - return qb; - } else if (qb instanceof final TermQueryBuilder query) { - final String translatedFieldName = USER_FIELD_NAME_TRANSLATOR.translate(query.fieldName()); - return QueryBuilders.termQuery(translatedFieldName, query.value()).caseInsensitive(query.caseInsensitive()); - } else if (qb instanceof final ExistsQueryBuilder query) { - final String translatedFieldName = USER_FIELD_NAME_TRANSLATOR.translate(query.fieldName()); - return QueryBuilders.existsQuery(translatedFieldName); - } else if (qb instanceof final TermsQueryBuilder query) { - if (query.termsLookup() != null) { - throw new IllegalArgumentException("Terms query with terms lookup is not supported for User query"); - } - final String translatedFieldName = USER_FIELD_NAME_TRANSLATOR.translate(query.fieldName()); - return QueryBuilders.termsQuery(translatedFieldName, query.getValues()); - } else if (qb instanceof final PrefixQueryBuilder query) { - final String translatedFieldName = USER_FIELD_NAME_TRANSLATOR.translate(query.fieldName()); - return QueryBuilders.prefixQuery(translatedFieldName, query.value()).caseInsensitive(query.caseInsensitive()); - } else if (qb instanceof final WildcardQueryBuilder query) { - final String translatedFieldName = USER_FIELD_NAME_TRANSLATOR.translate(query.fieldName()); - return QueryBuilders.wildcardQuery(translatedFieldName, query.value()) - .caseInsensitive(query.caseInsensitive()) - .rewrite(query.rewrite()); - } else { - throw new IllegalArgumentException("Query type [" + qb.getName() + "] is not supported for User query"); - } + return finalQuery; } @Override @@ -94,8 +52,7 @@ protected QueryBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws return super.doRewrite(queryRewriteContext); } - boolean isIndexFieldNameAllowed(String queryFieldName) { - // Type is needed to filter on user doc type - return queryFieldName.equals("type") || USER_FIELD_NAME_TRANSLATOR.supportedIndexFieldName(queryFieldName); + boolean isIndexFieldNameAllowed(String fieldName) { + return FIELDS_ALLOWED_TO_QUERY.contains(fieldName) || USER_FIELD_NAME_TRANSLATORS.isIndexFieldSupported(fieldName); } } diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy index 97b0f480043e5..2c9d38e5ae55e 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy @@ -1,6 +1,12 @@ grant { permission java.lang.RuntimePermission "setFactory"; + // secure the users file from other things (current and legacy locations) + permission org.elasticsearch.SecuredConfigFileAccessPermission "users"; + permission org.elasticsearch.SecuredConfigFileAccessPermission "x-pack/users"; + // other security files specified by settings + permission org.elasticsearch.SecuredConfigFileSettingAccessPermission "xpack.security.authc.realms.ldap.*.files.role_mapping"; + // needed for SAML permission java.util.PropertyPermission "org.apache.xml.security.ignoreLineBreaks", "read,write"; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyActionTests.java index 1593fadf1802d..09144e8f6edd5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/apikey/TransportQueryApiKeyActionTests.java @@ -13,13 +13,13 @@ import org.elasticsearch.search.sort.SortMode; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.security.support.ApiKeyFieldNameTranslators; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.IntStream; +import static org.elasticsearch.xpack.security.support.FieldNameTranslators.API_KEY_FIELD_NAME_TRANSLATORS; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -43,7 +43,7 @@ public void testTranslateFieldSortBuilders() { List sortFields = new ArrayList<>(); final SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource(); - ApiKeyFieldNameTranslators.translateFieldSortBuilders(originals, searchSourceBuilder, sortFields::add); + API_KEY_FIELD_NAME_TRANSLATORS.translateFieldSortBuilders(originals, searchSourceBuilder, sortFields::add); IntStream.range(0, originals.size()).forEach(i -> { final FieldSortBuilder original = originals.get(i); @@ -96,13 +96,13 @@ public void testNestedSortingIsNotAllowed() { fieldSortBuilder.setNestedSort(new NestedSortBuilder("name")); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> ApiKeyFieldNameTranslators.translateFieldSortBuilders( + () -> API_KEY_FIELD_NAME_TRANSLATORS.translateFieldSortBuilders( List.of(fieldSortBuilder), SearchSourceBuilder.searchSource(), ignored -> {} ) ); - assertThat(e.getMessage(), equalTo("nested sorting is not supported for API Key query")); + assertThat(e.getMessage(), equalTo("nested sorting is not currently supported in this context")); } private FieldSortBuilder randomFieldSortBuilderWithName(String name) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserActionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserActionTests.java index 3fb3a816baa8b..9d54d529f3cb9 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserActionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/action/user/TransportQueryUserActionTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.rest.RestStatus; @@ -43,13 +44,13 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import java.util.stream.IntStream; import static org.elasticsearch.test.ActionListenerUtils.anyActionListener; +import static org.elasticsearch.xpack.security.support.FieldNameTranslators.USER_FIELD_NAME_TRANSLATORS; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -72,7 +73,7 @@ public void testTranslateFieldSortBuilders() { final List originals = fieldNames.stream().map(this::randomFieldSortBuilderWithName).toList(); final SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.searchSource(); - TransportQueryUserAction.translateFieldSortBuilders(originals, searchSourceBuilder); + USER_FIELD_NAME_TRANSLATORS.translateFieldSortBuilders(originals, searchSourceBuilder, null); IntStream.range(0, originals.size()).forEach(i -> { final FieldSortBuilder original = originals.get(i); @@ -93,9 +94,13 @@ public void testNestedSortingIsNotAllowed() { fieldSortBuilder.setNestedSort(new NestedSortBuilder("something")); final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> TransportQueryUserAction.translateFieldSortBuilders(List.of(fieldSortBuilder), SearchSourceBuilder.searchSource()) + () -> USER_FIELD_NAME_TRANSLATORS.translateFieldSortBuilders( + List.of(fieldSortBuilder), + SearchSourceBuilder.searchSource(), + null + ) ); - assertThat(e.getMessage(), equalTo("nested sorting is not supported for User query")); + assertThat(e.getMessage(), equalTo("nested sorting is not currently supported in this context")); } public void testNestedSortingOnTextFieldsNotAllowed() { @@ -106,9 +111,9 @@ public void testNestedSortingOnTextFieldsNotAllowed() { final IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> TransportQueryUserAction.translateFieldSortBuilders(originals, searchSourceBuilder) + () -> USER_FIELD_NAME_TRANSLATORS.translateFieldSortBuilders(originals, searchSourceBuilder, null) ); - assertThat(e.getMessage(), equalTo(String.format(Locale.ROOT, "sorting is not supported for field [%s] in User query", fieldName))); + assertThat(e.getMessage(), equalTo(Strings.format("sorting is not supported for field [%s]", fieldName))); } public void testQueryUsers() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index e2fe682943a84..9d9528ec6f48b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -126,6 +126,7 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.threadpool.ThreadPool.Names; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.TransportActionProxy; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xcontent.XContentBuilder; @@ -227,7 +228,6 @@ import static org.hamcrest.Matchers.arrayContainingInAnyOrder; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -235,7 +235,6 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.hamcrest.Matchers.startsWith; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; import static org.mockito.ArgumentMatchers.eq; @@ -3158,41 +3157,38 @@ private ClusterState mockClusterState(Metadata metadata) { } public void testProxyRequestFailsOnNonProxyAction() { - TransportRequest request = TransportRequest.Empty.INSTANCE; + TransportRequest request = new EmptyRequest(); DiscoveryNode node = DiscoveryNodeUtils.create("foo"); TransportRequest transportRequest = TransportActionProxy.wrapRequest(node, request); - final String requestId = AuditUtil.getOrGenerateRequestId(threadContext); + AuditUtil.getOrGenerateRequestId(threadContext); User user = new User("test user", "role"); - ElasticsearchSecurityException ese = expectThrows( - ElasticsearchSecurityException.class, - () -> authorize(createAuthentication(user), "indices:some/action", transportRequest) - ); - assertThat(ese.getCause(), instanceOf(IllegalStateException.class)); - IllegalStateException illegalStateException = (IllegalStateException) ese.getCause(); - assertThat( - illegalStateException.getMessage(), - startsWith("originalRequest is a proxy request for: [org.elasticsearch.transport.TransportRequest$") + final var authentication = createAuthentication(user); + assertEquals( + """ + originalRequest is a proxy request for: [org.elasticsearch.transport.EmptyRequest/unset] \ + but action: [indices:some/action] isn't""", + expectThrows( + ElasticsearchSecurityException.class, + IllegalStateException.class, + () -> authorize(authentication, "indices:some/action", transportRequest) + ).getMessage() ); - assertThat(illegalStateException.getMessage(), endsWith("] but action: [indices:some/action] isn't")); } public void testProxyRequestFailsOnNonProxyRequest() { - TransportRequest request = TransportRequest.Empty.INSTANCE; + TransportRequest request = new EmptyRequest(); User user = new User("test user", "role"); AuditUtil.getOrGenerateRequestId(threadContext); - ElasticsearchSecurityException ese = expectThrows( - ElasticsearchSecurityException.class, - () -> authorize(createAuthentication(user), TransportActionProxy.getProxyAction("indices:some/action"), request) - ); - assertThat(ese.getCause(), instanceOf(IllegalStateException.class)); - IllegalStateException illegalStateException = (IllegalStateException) ese.getCause(); - assertThat( - illegalStateException.getMessage(), - startsWith("originalRequest is not a proxy request: [org.elasticsearch.transport.TransportRequest$") - ); - assertThat( - illegalStateException.getMessage(), - endsWith("] but action: [internal:transport/proxy/indices:some/action] is a proxy action") + final var authentication = createAuthentication(user); + assertEquals( + """ + originalRequest is not a proxy request: [org.elasticsearch.transport.EmptyRequest/unset] \ + but action: [internal:transport/proxy/indices:some/action] is a proxy action""", + expectThrows( + ElasticsearchSecurityException.class, + IllegalStateException.class, + () -> authorize(authentication, TransportActionProxy.getProxyAction("indices:some/action"), request) + ).getMessage() ); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java index fd2c0c7c6e8d8..e9408fd34c3ed 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizedIndicesTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationTestHelper; @@ -393,7 +394,7 @@ public void testDataStreamsAreIncludedInAuthorizedIndices() { } public static AuthorizationEngine.RequestInfo getRequestInfo(String action) { - return getRequestInfo(TransportRequest.Empty.INSTANCE, action); + return getRequestInfo(new EmptyRequest(), action); } public static AuthorizationEngine.RequestInfo getRequestInfo(TransportRequest request, String action) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index 82ac95a21086d..73a5ce8177153 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -58,6 +58,7 @@ import org.elasticsearch.search.internal.ShardSearchRequest; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.NoSuchRemoteClusterException; import org.elasticsearch.transport.TransportRequest; import org.elasticsearch.xpack.core.graph.action.GraphExploreAction; @@ -2541,7 +2542,7 @@ public void testResolveSearchShardRequestAgainstDataStream() { } private AuthorizedIndices buildAuthorizedIndices(User user, String action) { - return buildAuthorizedIndices(user, action, TransportRequest.Empty.INSTANCE); + return buildAuthorizedIndices(user, action, new EmptyRequest()); } private AuthorizedIndices buildAuthorizedIndices(User user, String action, TransportRequest request) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index c0b24a26010a7..5b28c3dc39cfe 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -49,8 +49,8 @@ import org.elasticsearch.test.MockLog; import org.elasticsearch.test.TransportVersionUtils; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.TransportRequest; -import org.elasticsearch.transport.TransportRequest.Empty; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.XPackPlugin; @@ -2079,7 +2079,7 @@ public void testApiKeyAuthUsesApiKeyServiceWithScopedRole() throws Exception { PlainActionFuture roleFuture = new PlainActionFuture<>(); compositeRolesStore.getRole(authentication.getEffectiveSubject(), roleFuture); Role role = roleFuture.actionGet(); - assertThat(role.checkClusterAction("cluster:admin/foo", Empty.INSTANCE, AuthenticationTestHelper.builder().build()), is(false)); + assertThat(role.checkClusterAction("cluster:admin/foo", new EmptyRequest(), AuthenticationTestHelper.builder().build()), is(false)); assertThat(effectiveRoleDescriptors.get(), is(nullValue())); if (version == TransportVersion.current()) { verify(apiKeyService).parseRoleDescriptorsBytes( diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java index fdc7b59528153..4a1f7daad2a37 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/ApiKeyBoolQueryBuilderTests.java @@ -51,7 +51,7 @@ import static org.elasticsearch.test.LambdaMatchers.falseWith; import static org.elasticsearch.test.LambdaMatchers.trueWith; -import static org.elasticsearch.xpack.security.support.ApiKeyFieldNameTranslators.FIELD_NAME_TRANSLATORS; +import static org.elasticsearch.xpack.security.support.FieldNameTranslators.API_KEY_FIELD_NAME_TRANSLATORS; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -133,12 +133,12 @@ public void testPrefixQueryBuilderPropertiesArePreserved() { } List queryFields = new ArrayList<>(); ApiKeyBoolQueryBuilder apiKeyMatchQueryBuilder = ApiKeyBoolQueryBuilder.build(prefixQueryBuilder, queryFields::add, authentication); - assertThat(queryFields, hasItem(ApiKeyFieldNameTranslators.translate(fieldName))); + assertThat(queryFields, hasItem(API_KEY_FIELD_NAME_TRANSLATORS.translate(fieldName))); List mustQueries = apiKeyMatchQueryBuilder.must(); assertThat(mustQueries, hasSize(1)); assertThat(mustQueries.get(0), instanceOf(PrefixQueryBuilder.class)); PrefixQueryBuilder prefixQueryBuilder2 = (PrefixQueryBuilder) mustQueries.get(0); - assertThat(prefixQueryBuilder2.fieldName(), is(ApiKeyFieldNameTranslators.translate(prefixQueryBuilder.fieldName()))); + assertThat(prefixQueryBuilder2.fieldName(), is(API_KEY_FIELD_NAME_TRANSLATORS.translate(prefixQueryBuilder.fieldName()))); assertThat(prefixQueryBuilder2.value(), is(prefixQueryBuilder.value())); assertThat(prefixQueryBuilder2.boost(), is(prefixQueryBuilder.boost())); assertThat(prefixQueryBuilder2.queryName(), is(prefixQueryBuilder.queryName())); @@ -267,7 +267,7 @@ public void testSimpleQueryBuilderPropertiesArePreserved() { assertThat(simpleQueryStringBuilder2.fields().size(), is(simpleQueryStringBuilder.fields().size())); for (Map.Entry fieldEntry : simpleQueryStringBuilder.fields().entrySet()) { assertThat( - simpleQueryStringBuilder2.fields().get(ApiKeyFieldNameTranslators.translate(fieldEntry.getKey())), + simpleQueryStringBuilder2.fields().get(API_KEY_FIELD_NAME_TRANSLATORS.translate(fieldEntry.getKey())), is(fieldEntry.getValue()) ); } @@ -341,12 +341,12 @@ public void testMatchQueryBuilderPropertiesArePreserved() { } List queryFields = new ArrayList<>(); ApiKeyBoolQueryBuilder apiKeyMatchQueryBuilder = ApiKeyBoolQueryBuilder.build(matchQueryBuilder, queryFields::add, authentication); - assertThat(queryFields, hasItem(ApiKeyFieldNameTranslators.translate(fieldName))); + assertThat(queryFields, hasItem(API_KEY_FIELD_NAME_TRANSLATORS.translate(fieldName))); List mustQueries = apiKeyMatchQueryBuilder.must(); assertThat(mustQueries, hasSize(1)); assertThat(mustQueries.get(0), instanceOf(MatchQueryBuilder.class)); MatchQueryBuilder matchQueryBuilder2 = (MatchQueryBuilder) mustQueries.get(0); - assertThat(matchQueryBuilder2.fieldName(), is(ApiKeyFieldNameTranslators.translate(matchQueryBuilder.fieldName()))); + assertThat(matchQueryBuilder2.fieldName(), is(API_KEY_FIELD_NAME_TRANSLATORS.translate(matchQueryBuilder.fieldName()))); assertThat(matchQueryBuilder2.value(), is(matchQueryBuilder.value())); assertThat(matchQueryBuilder2.operator(), is(matchQueryBuilder.operator())); assertThat(matchQueryBuilder2.analyzer(), is(matchQueryBuilder.analyzer())); @@ -612,7 +612,7 @@ public void testAllowListOfFieldNames() { final Authentication authentication = randomBoolean() ? AuthenticationTests.randomAuthentication(null, null) : null; final String randomFieldName = randomValueOtherThanMany( - s -> FIELD_NAME_TRANSLATORS.stream().anyMatch(t -> t.supports(s)), + API_KEY_FIELD_NAME_TRANSLATORS::isQueryFieldSupported, () -> randomAlphaOfLengthBetween(3, 20) ); final String fieldName = randomFrom( @@ -638,7 +638,7 @@ public void testAllowListOfFieldNames() { IllegalArgumentException.class, () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); - assertThat(e1.getMessage(), containsString("Field [" + fieldName + "] is not allowed for API Key query")); + assertThat(e1.getMessage(), containsString("Field [" + fieldName + "] is not allowed for querying")); } // also wrapped in a boolean query @@ -667,7 +667,7 @@ public void testAllowListOfFieldNames() { IllegalArgumentException.class, () -> ApiKeyBoolQueryBuilder.build(q2, ignored -> {}, authentication) ); - assertThat(e2.getMessage(), containsString("Field [" + fieldName + "] is not allowed for API Key query")); + assertThat(e2.getMessage(), containsString("Field [" + fieldName + "] is not allowed for querying")); } } @@ -678,7 +678,7 @@ public void testTermsLookupIsNotAllowed() { IllegalArgumentException.class, () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); - assertThat(e1.getMessage(), containsString("terms query with terms lookup is not supported for API Key query")); + assertThat(e1.getMessage(), containsString("terms query with terms lookup is not currently supported in this context")); } public void testRangeQueryWithRelationIsNotAllowed() { @@ -688,7 +688,7 @@ public void testRangeQueryWithRelationIsNotAllowed() { IllegalArgumentException.class, () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); - assertThat(e1.getMessage(), containsString("range query with relation is not supported for API Key query")); + assertThat(e1.getMessage(), containsString("range query with relation is not currently supported in this context")); } public void testDisallowedQueryTypes() { @@ -734,7 +734,7 @@ public void testDisallowedQueryTypes() { IllegalArgumentException.class, () -> ApiKeyBoolQueryBuilder.build(q1, ignored -> {}, authentication) ); - assertThat(e1.getMessage(), containsString("Query type [" + q1.getName() + "] is not supported for API Key query")); + assertThat(e1.getMessage(), containsString("Query type [" + q1.getName() + "] is not currently supported in this context")); // also wrapped in a boolean query { @@ -756,7 +756,7 @@ public void testDisallowedQueryTypes() { IllegalArgumentException.class, () -> ApiKeyBoolQueryBuilder.build(q2, ignored -> {}, authentication) ); - assertThat(e2.getMessage(), containsString("Query type [" + q1.getName() + "] is not supported for API Key query")); + assertThat(e2.getMessage(), containsString("Query type [" + q1.getName() + "] is not currently supported in this context")); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/UserBoolQueryBuilderTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/UserBoolQueryBuilderTests.java index 460980d318786..d2e53cbbe8684 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/UserBoolQueryBuilderTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/support/UserBoolQueryBuilderTests.java @@ -110,18 +110,14 @@ public void testAllowListOfFieldNames() { public void testTermsLookupIsNotAllowed() { final TermsQueryBuilder q1 = QueryBuilders.termsLookupQuery("roles", new TermsLookup("lookup", "1", "id")); final IllegalArgumentException e1 = expectThrows(IllegalArgumentException.class, () -> UserBoolQueryBuilder.build(q1)); - assertThat(e1.getMessage(), containsString("Terms query with terms lookup is not supported for User query")); + assertThat(e1.getMessage(), containsString("terms query with terms lookup is not currently supported in this context")); } public void testDisallowedQueryTypes() { final AbstractQueryBuilder> q1 = randomFrom( - QueryBuilders.idsQuery(), - QueryBuilders.rangeQuery(randomAlphaOfLength(5)), - QueryBuilders.matchQuery(randomAlphaOfLength(5), randomAlphaOfLength(5)), QueryBuilders.constantScoreQuery(mock(QueryBuilder.class)), QueryBuilders.boostingQuery(mock(QueryBuilder.class), mock(QueryBuilder.class)), QueryBuilders.queryStringQuery("q=a:42"), - QueryBuilders.simpleQueryStringQuery(randomAlphaOfLength(5)), QueryBuilders.combinedFieldsQuery(randomAlphaOfLength(5)), QueryBuilders.disMaxQuery(), QueryBuilders.distanceFeatureQuery( @@ -155,7 +151,7 @@ public void testDisallowedQueryTypes() { ); final IllegalArgumentException e1 = expectThrows(IllegalArgumentException.class, () -> UserBoolQueryBuilder.build(q1)); - assertThat(e1.getMessage(), containsString("Query type [" + q1.getName() + "] is not supported for User query")); + assertThat(e1.getMessage(), containsString("Query type [" + q1.getName() + "] is not currently supported in this context")); } public void testWillSetAllowedFields() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java index 981cae74f0530..3d3f96b98d5e5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/netty4/SecurityNetty4ServerTransportAuthenticationTests.java @@ -37,6 +37,7 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.BytesRefRecycler; import org.elasticsearch.transport.Compression; +import org.elasticsearch.transport.EmptyRequest; import org.elasticsearch.transport.ProxyConnectionStrategy; import org.elasticsearch.transport.RemoteClusterPortSettings; import org.elasticsearch.transport.RemoteClusterService; @@ -332,7 +333,7 @@ public void testConnectionDisconnectedWhenAuthnFails() throws Exception { try (Socket socket = new MockSocket(remoteIngressTransportAddress.getAddress(), remoteIngressTransportAddress.getPort())) { TestOutboundRequestMessage message = new TestOutboundRequestMessage( threadPool.getThreadContext(), - TransportRequest.Empty.INSTANCE, + new EmptyRequest(), TransportVersion.current(), "internal:whatever", randomNonNegativeLong(), diff --git a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java index 69043c606ef15..377016e80f386 100644 --- a/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java +++ b/x-pack/plugin/shutdown/src/main/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusAction.java @@ -49,7 +49,6 @@ import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -254,11 +253,20 @@ static ShutdownShardMigrationStatus shardMigrationStatus( int totalRemainingShards = relocatingShards + startedShards + initializingShards; // If there's relocating shards, or no shards on this node, we'll just use the number of shards left to move - if (relocatingShards > 0 || totalRemainingShards == 0) { - SingleNodeShutdownMetadata.Status shardStatus = totalRemainingShards == 0 - ? SingleNodeShutdownMetadata.Status.COMPLETE - : SingleNodeShutdownMetadata.Status.IN_PROGRESS; - return new ShutdownShardMigrationStatus(shardStatus, startedShards, relocatingShards, initializingShards); + if (totalRemainingShards == 0) { + return new ShutdownShardMigrationStatus( + SingleNodeShutdownMetadata.Status.COMPLETE, + startedShards, + relocatingShards, + initializingShards + ); + } else if (relocatingShards > 0) { + return new ShutdownShardMigrationStatus( + SingleNodeShutdownMetadata.Status.IN_PROGRESS, + startedShards, + relocatingShards, + initializingShards + ); } else if (initializingShards > 0 && relocatingShards == 0 && startedShards == 0) { // If there's only initializing shards left, return now with a note that only initializing shards are left return new ShutdownShardMigrationStatus( @@ -270,11 +278,8 @@ static ShutdownShardMigrationStatus shardMigrationStatus( ); } - // If there's no relocating shards and shards still on this node, we need to figure out why - AtomicInteger shardsToIgnoreForFinalStatus = new AtomicInteger(0); - - // Explain shard allocations until we find one that can't move, then stop (as `findFirst` short-circuits) - Optional> unmovableShard = currentState.getRoutingNodes() + // Get all shard explanations + var unmovableShards = currentState.getRoutingNodes() .node(nodeId) .shardsWithState(ShardRoutingState.STARTED) .peek(s -> cancellableTask.ensureNotCancelled()) @@ -285,10 +290,16 @@ static ShutdownShardMigrationStatus shardMigrationStatus( : "shard [" + pair + "] can remain on node [" + nodeId + "], but that node is shutting down"; return pair.v2().getMoveDecision().canRemain() == false; }) - // It's okay if some are throttled, they'll move eventually - .filter(pair -> pair.v2().getMoveDecision().getAllocationDecision().equals(AllocationDecision.THROTTLED) == false) // These shards will move as soon as possible .filter(pair -> pair.v2().getMoveDecision().getAllocationDecision().equals(AllocationDecision.YES) == false) + .toList(); + + // If there's no relocating shards and shards still on this node, we need to figure out why + AtomicInteger shardsToIgnoreForFinalStatus = new AtomicInteger(0); + + // Find first one that can not move permanently + var unmovableShard = unmovableShards.stream() + .filter(pair -> pair.v2().getMoveDecision().getAllocationDecision().equals(AllocationDecision.THROTTLED) == false) // If the shard that can't move is on every node in the cluster, we shouldn't be `STALLED` on it. .filter(pair -> { final boolean hasShardCopyOnOtherNode = hasShardCopyOnAnotherNode(currentState, pair.v1(), shuttingDownNodes); @@ -312,6 +323,10 @@ static ShutdownShardMigrationStatus shardMigrationStatus( ) .findFirst(); + var temporarilyUnmovableShards = unmovableShards.stream() + .filter(pair -> pair.v2().getMoveDecision().getAllocationDecision().equals(AllocationDecision.THROTTLED)) + .toList(); + if (totalRemainingShards == shardsToIgnoreForFinalStatus.get() && unmovableShard.isEmpty()) { return new ShutdownShardMigrationStatus( SingleNodeShutdownMetadata.Status.COMPLETE, @@ -338,14 +353,38 @@ static ShutdownShardMigrationStatus shardMigrationStatus( ), decision ); - } else { - return new ShutdownShardMigrationStatus( - SingleNodeShutdownMetadata.Status.IN_PROGRESS, - startedShards, - relocatingShards, - initializingShards - ); - } + } else if (relocatingShards == 0 + && initializingShards == 0 + && startedShards > 0 + && temporarilyUnmovableShards.size() == startedShards) { + // We found a shard that can't be moved temporarily, + // report it so that the cause of the throttling could be addressed if it is taking significant time + ShardRouting shardRouting = temporarilyUnmovableShards.get(0).v1(); + ShardAllocationDecision decision = temporarilyUnmovableShards.get(0).v2(); + + return new ShutdownShardMigrationStatus( + SingleNodeShutdownMetadata.Status.IN_PROGRESS, + startedShards, + relocatingShards, + initializingShards, + format( + "shard [%s] [%s] of index [%s] is waiting to be moved, see [%s] " + + "for details or use the cluster allocation explain API", + shardRouting.shardId().getId(), + shardRouting.primary() ? "primary" : "replica", + shardRouting.index().getName(), + NODE_ALLOCATION_DECISION_KEY + ), + decision + ); + } else { + return new ShutdownShardMigrationStatus( + SingleNodeShutdownMetadata.Status.IN_PROGRESS, + startedShards, + relocatingShards, + initializingShards + ); + } } private static boolean isIlmRestrictingShardMovement(ClusterState currentState, ShardRouting pair) { @@ -373,9 +412,8 @@ private static boolean isIlmRestrictingShardMovement(ClusterState currentState, private static boolean hasShardCopyOnAnotherNode(ClusterState clusterState, ShardRouting shardRouting, Set shuttingDownNodes) { return clusterState.routingTable() - .allShards(shardRouting.index().getName()) - .stream() - .filter(sr -> sr.id() == shardRouting.id()) + .shardRoutingTable(shardRouting.shardId()) + .allShards() .filter(sr -> sr.role().equals(shardRouting.role())) // If any shards are both 1) `STARTED` and 2) are not on a node that's shutting down, we have at least one copy // of this shard safely on a node that's not shutting down, so we don't want to report `STALLED` because of this shard. diff --git a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java index 9807fa72247a7..9a1dda99674c9 100644 --- a/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java +++ b/x-pack/plugin/shutdown/src/test/java/org/elasticsearch/xpack/shutdown/TransportGetShutdownStatusActionTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.routing.TestShardRouting; import org.elasticsearch.cluster.routing.UnassignedInfo; import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.routing.allocation.Explanations; import org.elasticsearch.cluster.routing.allocation.RoutingAllocation; import org.elasticsearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.elasticsearch.cluster.routing.allocation.decider.AllocationDecider; @@ -74,6 +75,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.doAnswer; @@ -410,7 +412,7 @@ public void testStalled() { status, SingleNodeShutdownMetadata.Status.STALLED, 1, - allOf(containsString(index.getName()), containsString("[2] [primary]")) + allOf(containsString(index.getName()), containsString("[2] [primary]"), containsString("cannot move")) ); } @@ -645,6 +647,49 @@ public void testNodeNotInCluster() { assertShardMigration(status, SingleNodeShutdownMetadata.Status.NOT_STARTED, 0, is("node is not currently part of the cluster")); } + public void testExplainThrottled() { + Index index = new Index(randomAlphaOfLength(5), randomAlphaOfLengthBetween(1, 20)); + IndexMetadata imd = generateIndexMetadata(index, 3, 0); + IndexRoutingTable indexRoutingTable = IndexRoutingTable.builder(index) + .addShard(TestShardRouting.newShardRouting(new ShardId(index, 0), LIVE_NODE_ID, true, ShardRoutingState.INITIALIZING)) + .addShard(TestShardRouting.newShardRouting(new ShardId(index, 1), LIVE_NODE_ID, true, ShardRoutingState.INITIALIZING)) + .addShard(TestShardRouting.newShardRouting(new ShardId(index, 2), SHUTTING_DOWN_NODE_ID, true, ShardRoutingState.STARTED)) + .build(); + + RoutingTable.Builder routingTable = RoutingTable.builder(); + routingTable.add(indexRoutingTable); + ClusterState state = createTestClusterState(routingTable.build(), List.of(imd), SingleNodeShutdownMetadata.Type.REMOVE); + + // LIVE_NODE_ID can not accept the remaining shard as it is temporarily initializing 2 other shards + canAllocate.set((r, n, a) -> n.nodeId().equals(LIVE_NODE_ID) ? Decision.THROTTLE : Decision.NO); + // And the remain decider simulates NodeShutdownAllocationDecider + canRemain.set((r, n, a) -> n.nodeId().equals(SHUTTING_DOWN_NODE_ID) ? Decision.NO : Decision.YES); + + ShutdownShardMigrationStatus status = TransportGetShutdownStatusAction.shardMigrationStatus( + new CancellableTask(1, "direct", GetShutdownStatusAction.NAME, "", TaskId.EMPTY_TASK_ID, Map.of()), + state, + SHUTTING_DOWN_NODE_ID, + SingleNodeShutdownMetadata.Type.REMOVE, + true, + clusterInfoService, + snapshotsInfoService, + allocationService, + allocationDeciders + ); + + assertShardMigration( + status, + SingleNodeShutdownMetadata.Status.IN_PROGRESS, + 1, + allOf(containsString(index.getName()), containsString("[2] [primary]"), containsString("is waiting to be moved")) + ); + var explain = status.getAllocationDecision(); + assertThat(explain, notNullValue()); + assertThat(explain.getAllocateDecision().isDecisionTaken(), is(false)); + assertThat(explain.getMoveDecision().isDecisionTaken(), is(true)); + assertThat(explain.getMoveDecision().getExplanation(), equalTo(Explanations.Move.THROTTLED)); + } + public void testIlmShrinkingIndexAvoidsStall() { LifecycleExecutionState executionState = LifecycleExecutionState.builder() .setAction(ShrinkAction.NAME) diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeParser.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeParser.java index 6e4593212716f..bbaf645aa0eb4 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeParser.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeParser.java @@ -16,7 +16,6 @@ import java.io.IOException; import java.text.ParseException; -import java.util.function.Consumer; class ShapeParser extends AbstractGeometryFieldMapper.Parser { private final GeometryParser geometryParser; @@ -26,18 +25,21 @@ class ShapeParser extends AbstractGeometryFieldMapper.Parser { } @Override - public void parse(XContentParser parser, CheckedConsumer consumer, Consumer onMalformed) - throws IOException { + public void parse( + XContentParser parser, + CheckedConsumer consumer, + AbstractGeometryFieldMapper.MalformedValueHandler malformedHandler + ) throws IOException { try { if (parser.currentToken() == XContentParser.Token.START_ARRAY) { while (parser.nextToken() != XContentParser.Token.END_ARRAY) { - parse(parser, consumer, onMalformed); + parse(parser, consumer, malformedHandler); } } else { consumer.accept(geometryParser.parse(parser)); } } catch (ParseException | ElasticsearchParseException | IllegalArgumentException e) { - onMalformed.accept(e); + malformedHandler.notify(e); } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilder.java index 843842cf863c7..8e73fc37f96ba 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilder.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilder.java @@ -9,6 +9,7 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.geo.LatLonGeometry; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticsearchParseException; @@ -284,7 +285,7 @@ public Query doToQuery(SearchExecutionContext context) { throw new QueryShardException(context, "failed to find geo field [" + fieldName + "]"); } } - return grid.toQuery(context, fieldName, fieldType, gridId); + return new ConstantScoreQuery(grid.toQuery(context, fieldName, fieldType, gridId)); } @Override diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index ad622109e1748..e4caa625e69df 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -489,7 +489,7 @@ private Value generateValue() { return new Value(nullValue, null); } - if (ignoreMalformed) { + if (ignoreMalformed && randomBoolean()) { // #exampleMalformedValues() covers a lot of cases // nice complex object diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java index 83eed8042e4de..5aff04520b5b8 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/GeoGridQueryBuilderTests.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.spatial.index.query; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -118,8 +119,11 @@ protected void doAssertLuceneQuery(GeoGridQueryBuilder queryBuilder, Query query final MappedFieldType fieldType = context.getFieldType(queryBuilder.fieldName()); if (fieldType == null) { assertTrue("Found no indexed geo query.", query instanceof MatchNoDocsQuery); - } else if (fieldType.hasDocValues()) { - assertEquals(IndexOrDocValuesQuery.class, query.getClass()); + } else { + assertEquals(ConstantScoreQuery.class, query.getClass()); + if (fieldType.hasDocValues()) { + assertEquals(IndexOrDocValuesQuery.class, ((ConstantScoreQuery) query).getQuery().getClass()); + } } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsRequest.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsRequest.java index af3a82905f8ee..9f05152b63158 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsRequest.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlStatsRequest.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.sql.plugin; -import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -34,11 +33,6 @@ public void includeStats(boolean includeStats) { this.includeStats = includeStats; } - @Override - public void writeTo(StreamOutput out) throws IOException { - TransportAction.localOnly(); - } - @Override public String toString() { return "sql_stats"; diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml index b719502ae8f28..726b9d153025e 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/analytics/histogram.yml @@ -306,3 +306,54 @@ histogram with large count values: - match: { aggregations.percent.values.1\.0: 0.2 } - match: { aggregations.percent.values.5\.0: 0.2 } - match: { aggregations.percent.values.25\.0: 0.2 } + +--- +histogram with synthetic source and ignore_malformed: + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: introduced in 8.15.0 + + - do: + indices.create: + index: histo_synthetic + body: + mappings: + _source: + mode: synthetic + properties: + latency: + type: histogram + ignore_malformed: true + + - do: + index: + index: histo_synthetic + id: "1" + body: + latency: "quick brown fox" + + - do: + index: + index: histo_synthetic + id: "2" + body: + latency: [{"values": [1.0], "counts": [1], "hello": "world"}, [123, 456], {"values": [2.0], "counts": [2]}, "fox"] + + - do: + indices.refresh: {} + + - do: + get: + index: histo_synthetic + id: 1 + - match: + _source: + latency: "quick brown fox" + + - do: + get: + index: histo_synthetic + id: 2 + - match: + _source: + latency: [{"values": [2.0], "counts": [2]}, {"values": [1.0], "counts": [1], "hello": "world"}, 123, 456, "fox"] diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml index 5f76954e57c89..96bfabf862f50 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/150_lookup.yml @@ -38,7 +38,7 @@ basic: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -66,7 +66,7 @@ read multivalue keyword: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -98,7 +98,7 @@ keyword matches text: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -144,7 +144,7 @@ duplicate keys: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -167,7 +167,7 @@ multivalued keys: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: @@ -209,7 +209,7 @@ on function: - method: POST path: /_query parameters: [] - capabilities: [lookup_command, tables_types] + capabilities: [tables_types] reason: "uses LOOKUP" - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml new file mode 100644 index 0000000000000..f3403ca8751c0 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml @@ -0,0 +1,573 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_query + parameters: [method, path, parameters, capabilities] + capabilities: [union_types] + reason: "Union types introduced in 8.15.0" + test_runner_features: [capabilities, allowed_warnings_regex] + + - do: + indices.create: + index: events_ip_long + body: + mappings: + properties: + "@timestamp": + type: date + client_ip: + type: ip + event_duration: + type: long + message: + type: keyword + + - do: + bulk: + refresh: true + index: events_ip_long + body: + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:55:01.543Z", "client_ip": "172.21.3.15", "event_duration": 1756467, "message": "Connected to 10.1.0.1"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:53:55.832Z", "client_ip": "172.21.3.15", "event_duration": 5033755, "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:52:55.015Z", "client_ip": "172.21.3.15", "event_duration": 8268153, "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:51:54.732Z", "client_ip": "172.21.3.15", "event_duration": 725448, "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:33:34.937Z", "client_ip": "172.21.0.5", "event_duration": 1232382, "message": "Disconnected"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T12:27:28.948Z", "client_ip": "172.21.2.113", "event_duration": 2764889, "message": "Connected to 10.1.0.2"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T12:15:03.360Z", "client_ip": "172.21.2.162", "event_duration": 3450233, "message": "Connected to 10.1.0.3"}' + - do: + indices.create: + index: events_keyword_long + body: + mappings: + properties: + "@timestamp": + type: date + client_ip: + type: keyword + event_duration: + type: long + message: + type: keyword + + - do: + bulk: + refresh: true + index: events_keyword_long + body: + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:55:01.543Z", "client_ip": "172.21.3.15", "event_duration": 1756467, "message": "Connected to 10.1.0.1"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:53:55.832Z", "client_ip": "172.21.3.15", "event_duration": 5033755, "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:52:55.015Z", "client_ip": "172.21.3.15", "event_duration": 8268153, "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:51:54.732Z", "client_ip": "172.21.3.15", "event_duration": 725448, "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:33:34.937Z", "client_ip": "172.21.0.5", "event_duration": 1232382, "message": "Disconnected"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T12:27:28.948Z", "client_ip": "172.21.2.113", "event_duration": 2764889, "message": "Connected to 10.1.0.2"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T12:15:03.360Z", "client_ip": "172.21.2.162", "event_duration": 3450233, "message": "Connected to 10.1.0.3"}' + + - do: + indices.create: + index: events_ip_keyword + body: + mappings: + properties: + "@timestamp": + type: date + client_ip: + type: ip + event_duration: + type: keyword + message: + type: keyword + + - do: + bulk: + refresh: true + index: events_ip_keyword + body: + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:55:01.543Z", "client_ip": "172.21.3.15", "event_duration": "1756467", "message": "Connected to 10.1.0.1"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:53:55.832Z", "client_ip": "172.21.3.15", "event_duration": "5033755", "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:52:55.015Z", "client_ip": "172.21.3.15", "event_duration": "8268153", "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:51:54.732Z", "client_ip": "172.21.3.15", "event_duration": "725448", "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:33:34.937Z", "client_ip": "172.21.0.5", "event_duration": "1232382", "message": "Disconnected"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T12:27:28.948Z", "client_ip": "172.21.2.113", "event_duration": "2764889", "message": "Connected to 10.1.0.2"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T12:15:03.360Z", "client_ip": "172.21.2.162", "event_duration": "3450233", "message": "Connected to 10.1.0.3"}' + + - do: + indices.create: + index: events_keyword_keyword + body: + mappings: + properties: + "@timestamp": + type: date + client_ip: + type: keyword + event_duration: + type: keyword + message: + type: keyword + + - do: + bulk: + refresh: true + index: events_keyword_keyword + body: + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:55:01.543Z", "client_ip": "172.21.3.15", "event_duration": "1756467", "message": "Connected to 10.1.0.1"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:53:55.832Z", "client_ip": "172.21.3.15", "event_duration": "5033755", "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:52:55.015Z", "client_ip": "172.21.3.15", "event_duration": "8268153", "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:51:54.732Z", "client_ip": "172.21.3.15", "event_duration": "725448", "message": "Connection error"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T13:33:34.937Z", "client_ip": "172.21.0.5", "event_duration": "1232382", "message": "Disconnected"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T12:27:28.948Z", "client_ip": "172.21.2.113", "event_duration": "2764889", "message": "Connected to 10.1.0.2"}' + - '{"index": {}}' + - '{"@timestamp": "2023-10-23T12:15:03.360Z", "client_ip": "172.21.2.162", "event_duration": "3450233", "message": "Connected to 10.1.0.3"}' + +--- +load single index ip_long: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_ip_long METADATA _index | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "ip" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "long" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 7 } + - match: { values.0.0: "events_ip_long" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: "172.21.3.15" } + - match: { values.0.3: 1756467 } + - match: { values.0.4: "Connected to 10.1.0.1" } + +############################################################################################################ +# Test a single index as a control of the expected results + +--- +load single index keyword_keyword: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_keyword_keyword METADATA _index | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "keyword" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "keyword" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 7 } + - match: { values.0.0: "events_keyword_keyword" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: "172.21.3.15" } + - match: { values.0.3: "1756467" } + - match: { values.0.4: "Connected to 10.1.0.1" } + +############################################################################################################ +# Test two indices where the event_duration is mapped as a LONG and as a KEYWORD + +--- +load two indices, showing unsupported type and null value for event_duration: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_ip_* METADATA _index | SORT _index ASC, @timestamp DESC' + + - length: { values: 14 } + + - match: { columns.0.name: "@timestamp" } + - match: { columns.0.type: "date" } + - match: { columns.1.name: "client_ip" } + - match: { columns.1.type: "ip" } + - match: { columns.2.name: "event_duration" } + - match: { columns.2.type: "unsupported" } + - match: { columns.3.name: "message" } + - match: { columns.3.type: "keyword" } + - match: { columns.4.name: "_index" } + - match: { columns.4.type: "keyword" } + - length: { values: 14 } + - match: { values.0.0: "2023-10-23T13:55:01.543Z" } + - match: { values.0.1: "172.21.3.15" } + - match: { values.0.2: null } + - match: { values.0.3: "Connected to 10.1.0.1" } + - match: { values.0.4: "events_ip_keyword" } + - match: { values.7.0: "2023-10-23T13:55:01.543Z" } + - match: { values.7.1: "172.21.3.15" } + - match: { values.7.2: null } + - match: { values.7.3: "Connected to 10.1.0.1" } + - match: { values.7.4: "events_ip_long" } + +--- +load two indices with no conversion function, but needs TO_LONG conversion: + - do: + catch: '/Cannot use field \[event_duration\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] in \[events_ip_keyword\], \[long\] in \[events_ip_long\]/' + esql.query: + body: + query: 'FROM events_ip_* METADATA _index | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + +--- +load two indices with incorrect conversion function, TO_IP instead of TO_LONG: + - do: + catch: '/Cannot use field \[event_duration\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] in \[events_ip_keyword\], \[long\] in \[events_ip_long\]/' + esql.query: + body: + query: 'FROM events_ip_* METADATA _index | EVAL event_duration = TO_IP(event_duration) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + +--- +load two indices with single conversion function TO_LONG: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_ip_* METADATA _index | EVAL event_duration = TO_LONG(event_duration) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "ip" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "long" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 14 } + - match: { values.0.0: "events_ip_keyword" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: "172.21.3.15" } + - match: { values.0.3: 1756467 } + - match: { values.0.4: "Connected to 10.1.0.1" } + - match: { values.7.0: "events_ip_long" } + - match: { values.7.1: "2023-10-23T13:55:01.543Z" } + - match: { values.7.2: "172.21.3.15" } + - match: { values.7.3: 1756467 } + - match: { values.7.4: "Connected to 10.1.0.1" } + +--- +load two indices and drop ambiguous field event_duration: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_ip_* | DROP event_duration' + + - length: { values: 14 } + +--- +load two indices, convert and then drop ambiguous field event_duration: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_ip_* | EVAL event_duration = TO_LONG(event_duration) | DROP event_duration' + + - length: { values: 14 } + +--- +load two indices, convert, rename and then drop ambiguous field event_duration: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_ip_* | EVAL x = TO_LONG(event_duration) | DROP event_duration' + + - length: { values: 14 } + +--- +# This test needs to change to produce unsupported/null for the original field name +load two indices, convert, rename but not drop ambiguous field event_duration: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_ip_* | EVAL x = TO_LONG(event_duration), y = TO_STRING(event_duration), z = TO_LONG(event_duration) | SORT @timestamp DESC' + + - match: { columns.0.name: "@timestamp" } + - match: { columns.0.type: "date" } + - match: { columns.1.name: "client_ip" } + - match: { columns.1.type: "ip" } + - match: { columns.2.name: "event_duration" } + - match: { columns.2.type: "unsupported" } + - match: { columns.3.name: "message" } + - match: { columns.3.type: "keyword" } + - match: { columns.4.name: "x" } + - match: { columns.4.type: "long" } + - match: { columns.5.name: "y" } + - match: { columns.5.type: "keyword" } + - match: { columns.6.name: "z" } + - match: { columns.6.type: "long" } + - length: { values: 14 } + - match: { values.0.0: "2023-10-23T13:55:01.543Z" } + - match: { values.0.1: "172.21.3.15" } + - match: { values.0.2: null } + - match: { values.0.3: "Connected to 10.1.0.1" } + - match: { values.0.4: 1756467 } + - match: { values.0.5: "1756467" } + - match: { values.0.6: 1756467 } + - match: { values.1.0: "2023-10-23T13:55:01.543Z" } + - match: { values.1.1: "172.21.3.15" } + - match: { values.1.2: null } + - match: { values.1.3: "Connected to 10.1.0.1" } + - match: { values.1.4: 1756467 } + - match: { values.1.5: "1756467" } + - match: { values.1.6: 1756467 } + +############################################################################################################ +# Test two indices where the IP address is mapped as an IP and as a KEYWORD + +--- +load two indices, showing unsupported type and null value for client_ip: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_*_long METADATA _index | SORT _index ASC, @timestamp DESC' + + - match: { columns.0.name: "@timestamp" } + - match: { columns.0.type: "date" } + - match: { columns.1.name: "client_ip" } + - match: { columns.1.type: "unsupported" } + - match: { columns.2.name: "event_duration" } + - match: { columns.2.type: "long" } + - match: { columns.3.name: "message" } + - match: { columns.3.type: "keyword" } + - match: { columns.4.name: "_index" } + - match: { columns.4.type: "keyword" } + - length: { values: 14 } + - match: { values.0.0: "2023-10-23T13:55:01.543Z" } + - match: { values.0.1: null } + - match: { values.0.2: 1756467 } + - match: { values.0.3: "Connected to 10.1.0.1" } + - match: { values.0.4: "events_ip_long" } + - match: { values.7.0: "2023-10-23T13:55:01.543Z" } + - match: { values.7.1: null } + - match: { values.7.2: 1756467 } + - match: { values.7.3: "Connected to 10.1.0.1" } + - match: { values.7.4: "events_keyword_long" } + +--- +load two indices with no conversion function, but needs TO_IP conversion: + - do: + catch: '/Cannot use field \[client_ip\] due to ambiguities being mapped as \[2\] incompatible types: \[ip\] in \[events_ip_long\], \[keyword\] in \[events_keyword_long\]/' + esql.query: + body: + query: 'FROM events_*_long METADATA _index | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + +--- +load two indices with incorrect conversion function, TO_LONG instead of TO_IP: + - do: + catch: '/Cannot use field \[client_ip\] due to ambiguities being mapped as \[2\] incompatible types: \[ip\] in \[events_ip_long\], \[keyword\] in \[events_keyword_long\]/' + esql.query: + body: + query: 'FROM events_*_long METADATA _index | EVAL client_ip = TO_LONG(client_ip) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + +--- +load two indices with single conversion function TO_IP: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_*_long METADATA _index | EVAL client_ip = TO_IP(client_ip) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "ip" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "long" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 14 } + - match: { values.0.0: "events_ip_long" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: "172.21.3.15" } + - match: { values.0.3: 1756467 } + - match: { values.0.4: "Connected to 10.1.0.1" } + - match: { values.7.0: "events_keyword_long" } + - match: { values.7.1: "2023-10-23T13:55:01.543Z" } + - match: { values.7.2: "172.21.3.15" } + - match: { values.7.3: 1756467 } + - match: { values.7.4: "Connected to 10.1.0.1" } + +--- +load two indices and drop ambiguous field client_ip: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_*_long | DROP client_ip' + + - length: { values: 14 } + +--- +load two indices, convert and then drop ambiguous field client_ip: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_*_long | EVAL client_ip = TO_IP(client_ip) | DROP client_ip' + + - length: { values: 14 } + +--- +load two indices, convert, rename and then drop ambiguous field client_ip: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_*_long | EVAL x = TO_IP(client_ip) | DROP client_ip' + + - length: { values: 14 } + +--- +# This test needs to change to produce unsupported/null for the original field name +load two indices, convert, rename but not drop ambiguous field client_ip: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_*_long | EVAL x = TO_IP(client_ip), y = TO_STRING(client_ip), z = TO_IP(client_ip) | SORT @timestamp DESC' + + - match: { columns.0.name: "@timestamp" } + - match: { columns.0.type: "date" } + - match: { columns.1.name: "client_ip" } + - match: { columns.1.type: "unsupported" } + - match: { columns.2.name: "event_duration" } + - match: { columns.2.type: "long" } + - match: { columns.3.name: "message" } + - match: { columns.3.type: "keyword" } + - match: { columns.4.name: "x" } + - match: { columns.4.type: "ip" } + - match: { columns.5.name: "y" } + - match: { columns.5.type: "keyword" } + - match: { columns.6.name: "z" } + - match: { columns.6.type: "ip" } + - length: { values: 14 } + - match: { values.0.0: "2023-10-23T13:55:01.543Z" } + - match: { values.0.1: null } + - match: { values.0.2: 1756467 } + - match: { values.0.3: "Connected to 10.1.0.1" } + - match: { values.0.4: "172.21.3.15" } + - match: { values.0.5: "172.21.3.15" } + - match: { values.0.6: "172.21.3.15" } + - match: { values.1.0: "2023-10-23T13:55:01.543Z" } + - match: { values.1.1: null } + - match: { values.1.2: 1756467 } + - match: { values.1.3: "Connected to 10.1.0.1" } + - match: { values.1.4: "172.21.3.15" } + - match: { values.1.5: "172.21.3.15" } + - match: { values.1.6: "172.21.3.15" } + +############################################################################################################ +# Test four indices with both the client_IP (IP and KEYWORD) and event_duration (LONG and KEYWORD) mappings + +--- +load four indices with single conversion function TO_LONG: + - do: + catch: '/Cannot use field \[client_ip\] due to ambiguities being mapped as \[2\] incompatible types: \[ip\] in \[events_ip_keyword, events_ip_long\], \[keyword\] in \[events_keyword_keyword, events_keyword_long\]/' + esql.query: + body: + query: 'FROM events_* METADATA _index | EVAL event_duration = TO_LONG(event_duration) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + +--- +load four indices with single conversion function TO_IP: + - do: + catch: '/Cannot use field \[event_duration\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] in \[events_ip_keyword, events_keyword_keyword\], \[long\] in \[events_ip_long, events_keyword_long\]/' + esql.query: + body: + query: 'FROM events_* METADATA _index | EVAL client_ip = TO_IP(client_ip) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + +--- +load four indices with multiple conversion functions TO_LONG and TO_IP: + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM events_* METADATA _index | EVAL event_duration = TO_LONG(event_duration), client_ip = TO_IP(client_ip) | KEEP _index, @timestamp, client_ip, event_duration, message | SORT _index ASC, @timestamp DESC' + + - match: { columns.0.name: "_index" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "@timestamp" } + - match: { columns.1.type: "date" } + - match: { columns.2.name: "client_ip" } + - match: { columns.2.type: "ip" } + - match: { columns.3.name: "event_duration" } + - match: { columns.3.type: "long" } + - match: { columns.4.name: "message" } + - match: { columns.4.type: "keyword" } + - length: { values: 28 } + - match: { values.0.0: "events_ip_keyword" } + - match: { values.0.1: "2023-10-23T13:55:01.543Z" } + - match: { values.0.2: "172.21.3.15" } + - match: { values.0.3: 1756467 } + - match: { values.0.4: "Connected to 10.1.0.1" } + - match: { values.7.0: "events_ip_long" } + - match: { values.7.1: "2023-10-23T13:55:01.543Z" } + - match: { values.7.2: "172.21.3.15" } + - match: { values.7.3: 1756467 } + - match: { values.7.4: "Connected to 10.1.0.1" } + - match: { values.14.0: "events_keyword_keyword" } + - match: { values.14.1: "2023-10-23T13:55:01.543Z" } + - match: { values.14.2: "172.21.3.15" } + - match: { values.14.3: 1756467 } + - match: { values.14.4: "Connected to 10.1.0.1" } + - match: { values.21.0: "events_keyword_long" } + - match: { values.21.1: "2023-10-23T13:55:01.543Z" } + - match: { values.21.2: "172.21.3.15" } + - match: { values.21.3: 1756467 } + - match: { values.21.4: "Connected to 10.1.0.1" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/161_union_types_subfields.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/161_union_types_subfields.yml new file mode 100644 index 0000000000000..99bd1d6508895 --- /dev/null +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/161_union_types_subfields.yml @@ -0,0 +1,203 @@ +setup: + - requires: + capabilities: + - method: POST + path: /_query + parameters: [ method, path, parameters, capabilities ] + capabilities: [ union_types ] + reason: "Union types introduced in 8.15.0" + test_runner_features: [ capabilities, allowed_warnings_regex ] + + - do: + indices.create: + index: test1 + body: + mappings: + properties: + obj: + properties: + keyword: + type: keyword + integer: + type: integer + keyword: + type: boolean + integer: + type: version + + - do: + indices.create: + index: test2 + body: + mappings: + properties: + obj: + properties: + keyword: + type: boolean + integer: + type: version + keyword: + type: keyword + integer: + type: integer + + - do: + bulk: + refresh: true + index: test1 + body: + - '{ "index": {"_id": 11} }' + - '{ "obj.keyword": "true", "obj.integer": 100, "keyword": "true", "integer": "50" }' + - '{ "index": {"_id": 12} }' + - '{ "obj.keyword": "US", "obj.integer": 20, "keyword": false, "integer": "1.2.3" }' + + - do: + bulk: + refresh: true + index: test2 + body: + - '{ "index": {"_id": 21} }' + - '{ "obj.keyword": "true", "obj.integer": "50", "keyword": "true", "integer": 100 }' + - '{ "index": {"_id": 22} }' + - '{ "obj.keyword": false, "obj.integer": "1.2.3", "keyword": "US", "integer": 20 }' + +--- +"load single index": + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test1 METADATA _id | KEEP _id, obj.integer, obj.keyword | SORT _id ASC' + + - match: { columns.0.name: "_id" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "obj.integer" } + - match: { columns.1.type: "integer" } + - match: { columns.2.name: "obj.keyword" } + - match: { columns.2.type: "keyword" } + - length: { values: 2 } + - match: { values.0.0: "11" } + - match: { values.0.1: 100 } + - match: { values.0.2: "true" } + - match: { values.1.0: "12" } + - match: { values.1.1: 20 } + - match: { values.1.2: "US" } + +--- +"load two indices with to_string": + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test* METADATA _id | EVAL s = TO_STRING(obj.keyword) | KEEP _id, s | SORT _id ASC' + + - match: { columns.0.name: "_id" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "s" } + - match: { columns.1.type: "keyword" } + - length: { values: 4 } + - match: { values.0.0: "11" } + - match: { values.0.1: "true" } + - match: { values.1.0: "12" } + - match: { values.1.1: "US" } + - match: { values.2.0: "21" } + - match: { values.2.1: "true" } + - match: { values.3.0: "22" } + - match: { values.3.1: "false" } + + +--- +"load two indices with to_version": + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test* METADATA _id | EVAL v = TO_VERSION(TO_STRING(obj.integer)) | KEEP _id, v | SORT _id ASC' + + - match: { columns.0.name: "_id" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "v" } + - match: { columns.1.type: "version" } + - length: { values: 4 } + - match: { values.0.0: "11" } + - match: { values.0.1: "100" } + - match: { values.1.0: "12" } + - match: { values.1.1: "20" } + - match: { values.2.0: "21" } + - match: { values.2.1: "50" } + - match: { values.3.0: "22" } + - match: { values.3.1: "1.2.3" } + +--- +"load two indices with to_version and to_string": + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test* METADATA _id | EVAL v = TO_VERSION(TO_STRING(obj.integer)), s = TO_STRING(obj.keyword) | KEEP _id, v, s | SORT _id ASC' + + - match: { columns.0.name: "_id" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "v" } + - match: { columns.1.type: "version" } + - match: { columns.2.name: "s" } + - match: { columns.2.type: "keyword" } + - length: { values: 4 } + - match: { values.0.0: "11" } + - match: { values.0.1: "100" } + - match: { values.0.2: "true" } + - match: { values.1.0: "12" } + - match: { values.1.1: "20" } + - match: { values.1.2: "US" } + - match: { values.2.0: "21" } + - match: { values.2.1: "50" } + - match: { values.2.2: "true" } + - match: { values.3.0: "22" } + - match: { values.3.1: "1.2.3" } + - match: { values.3.2: "false" } + +--- +"load two indices with to_version and to_string nested and un-nested": + - do: + allowed_warnings_regex: + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test* METADATA _id | EVAL nv = TO_VERSION(TO_STRING(obj.integer)), uv = TO_VERSION(TO_STRING(integer)), ns = TO_STRING(obj.keyword), us = TO_STRING(keyword) | KEEP _id, nv, uv, ns, us | SORT _id ASC' + + - match: { columns.0.name: "_id" } + - match: { columns.0.type: "keyword" } + - match: { columns.1.name: "nv" } + - match: { columns.1.type: "version" } + - match: { columns.2.name: "uv" } + - match: { columns.2.type: "version" } + - match: { columns.3.name: "ns" } + - match: { columns.3.type: "keyword" } + - match: { columns.4.name: "us" } + - match: { columns.4.type: "keyword" } + - length: { values: 4 } + - match: { values.0.0: "11" } + - match: { values.0.1: "100" } + - match: { values.0.2: "50" } + - match: { values.0.3: "true" } + - match: { values.0.4: "true" } + - match: { values.1.0: "12" } + - match: { values.1.1: "20" } + - match: { values.1.2: "1.2.3" } + - match: { values.1.3: "US" } + - match: { values.1.4: "false" } + - match: { values.2.0: "21" } + - match: { values.2.1: "50" } + - match: { values.2.2: "100" } + - match: { values.2.3: "true" } + - match: { values.2.4: "true" } + - match: { values.3.0: "22" } + - match: { values.3.1: "1.2.3" } + - match: { values.3.2: "20" } + - match: { values.3.3: "false" } + - match: { values.3.4: "US" } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml index ccc6cd8627b53..700142cec9987 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/spatial/140_synthetic_source.yml @@ -411,6 +411,86 @@ - match: { _source.point.lon: -71.34000029414892 } - match: { _source.point.lat: 41.119999922811985 } +--- +"geo_point with ignore_malformed": + - requires: + cluster_features: ["mapper.track_ignored_source"] + reason: introduced in 8.15.0 + test_runner_features: close_to + + - do: + indices.create: + index: test + body: + mappings: + _source: + mode: synthetic + properties: + geo_point: + type: geo_point + ignore_malformed: true + + - do: + index: + index: test + id: "1" + body: + geo_point: + - string: "string" + array: [{ "a": 1 }, { "b": 2 }] + object: { "foo": "bar" } + - lat: 41.12 + lon: -71.34 + + - do: + index: + index: test + id: "2" + body: + geo_point: ["POINT (-71.34 41.12)", "potato", "POINT (-77.03653 38.897676)"] + + - do: + index: + index: test + id: "3" + body: + geo_point: ["POINT (-77.03653 1000)", "POINT (-71.34 41.12)"] + + - do: + indices.refresh: {} + + - do: + get: + index: test + id: "1" + + - close_to: { _source.geo_point.0.lon: { value: -71.34, error: 0.001 } } + - close_to: { _source.geo_point.0.lat: { value: 41.12, error: 0.001 } } + - match: { _source.geo_point.1.string: "string" } + - match: { _source.geo_point.1.array: [{ "a": 1 }, { "b": 2 }] } + - match: { _source.geo_point.1.object: { "foo": "bar" } } + + - do: + get: + index: test + id: "2" + + - close_to: { _source.geo_point.0.lon: { value: -77.03653, error: 0.0001 } } + - close_to: { _source.geo_point.0.lat: { value: 38.897676, error: 0.0001 } } + - close_to: { _source.geo_point.1.lon: { value: -71.34, error: 0.001 } } + - close_to: { _source.geo_point.1.lat: { value: 41.12, error: 0.001 } } + - match: { _source.geo_point.2: "potato" } + + - do: + get: + index: test + id: "3" + + - close_to: { _source.geo_point.0.lon: { value: -71.34, error: 0.001 } } + - close_to: { _source.geo_point.0.lat: { value: 41.12, error: 0.001 } } + - match: { _source.geo_point.1: "POINT (-77.03653 1000)" } + + --- "point": - requires: diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java index 2577cf28f4213..cc127883652af 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java @@ -12,7 +12,6 @@ import org.elasticsearch.plugins.Plugin; import java.util.Collection; -import java.util.Collections; import java.util.List; public class StackPlugin extends Plugin implements ActionPlugin { @@ -24,7 +23,7 @@ public StackPlugin(Settings settings) { @Override public List> getSettings() { - return Collections.singletonList(StackTemplateRegistry.STACK_TEMPLATES_ENABLED); + return List.of(StackTemplateRegistry.STACK_TEMPLATES_ENABLED, StackTemplateRegistry.CLUSTER_LOGSDB_ENABLED); } @Override diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index 3cd551ca1f3d9..34cacbb8956e5 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -57,10 +57,21 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { Setting.Property.Dynamic ); + /** + * if index.mode "logs" is applied by default in logs@settings for 'logs-*-*' + */ + public static final Setting CLUSTER_LOGSDB_ENABLED = Setting.boolSetting( + "cluster.logsdb.enabled", + false, + Setting.Property.NodeScope + ); + private final ClusterService clusterService; private final FeatureService featureService; private volatile boolean stackTemplateEnabled; + private final boolean logsIndexModeTemplateEnabled; + public static final Map ADDITIONAL_TEMPLATE_VARIABLES = Map.of("xpack.stack.template.deprecated", "false"); // General mappings conventions for any data that ends up in a data stream @@ -121,6 +132,7 @@ public StackTemplateRegistry( this.clusterService = clusterService; this.featureService = featureService; this.stackTemplateEnabled = STACK_TEMPLATES_ENABLED.get(nodeSettings); + this.logsIndexModeTemplateEnabled = CLUSTER_LOGSDB_ENABLED.get(nodeSettings); } @Override @@ -164,6 +176,7 @@ protected List getLifecyclePolicies() { } private static final Map COMPONENT_TEMPLATE_CONFIGS; + private static final Map LOGSDB_COMPONENT_TEMPLATE_CONFIGS; static { final Map componentTemplates = new HashMap<>(); @@ -249,10 +262,97 @@ protected List getLifecyclePolicies() { } } COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(componentTemplates); + + final Map logsdbComponentTemplates = new HashMap<>(); + for (IndexTemplateConfig config : List.of( + new IndexTemplateConfig( + DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/data-streams@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/logs@mappings-logsdb.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + ECS_DYNAMIC_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/ecs@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/logs@settings-logsdb.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + METRICS_TSDB_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/metrics@tsdb-settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/synthetics@mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/synthetics@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ), + new IndexTemplateConfig( + KIBANA_REPORTING_COMPONENT_TEMPLATE_NAME, + "/kibana-reporting@settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE, + ADDITIONAL_TEMPLATE_VARIABLES + ) + )) { + try { + logsdbComponentTemplates.put( + config.getTemplateName(), + ComponentTemplate.parse(JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, config.loadBytes())) + ); + } catch (IOException e) { + throw new AssertionError(e); + } + } + LOGSDB_COMPONENT_TEMPLATE_CONFIGS = Map.copyOf(logsdbComponentTemplates); } @Override protected Map getComponentTemplateConfigs() { + if (logsIndexModeTemplateEnabled) { + return LOGSDB_COMPONENT_TEMPLATE_CONFIGS; + } return COMPONENT_TEMPLATE_CONFIGS; } diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookTokenIntegrationTests.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookTokenIntegrationTests.java index 7da2c5b718356..611c32c48fec5 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookTokenIntegrationTests.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/actions/webhook/WebhookTokenIntegrationTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.action.admin.cluster.node.reload.NodesReloadSecureSettingsRequest; import org.elasticsearch.action.admin.cluster.node.reload.TransportNodesReloadSecureSettingsAction; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.KeyStoreWrapper; import org.elasticsearch.common.settings.MockSecureSettings; import org.elasticsearch.common.settings.SecureString; @@ -99,7 +100,7 @@ public void testWebhook() throws Exception { ksw.save(configPath, "".toCharArray(), false); } // Reload the keystore to load the new settings - NodesReloadSecureSettingsRequest reloadReq = new NodesReloadSecureSettingsRequest(); + NodesReloadSecureSettingsRequest reloadReq = new NodesReloadSecureSettingsRequest(Strings.EMPTY_ARRAY); try { reloadReq.setSecureStorePassword(new SecureString("".toCharArray())); client().execute(TransportNodesReloadSecureSettingsAction.TYPE, reloadReq).get(); diff --git a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java index 44b1a6ce51b50..c28e5f1e0fce8 100644 --- a/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java +++ b/x-pack/qa/evil-tests/src/test/java/org/elasticsearch/xpack/security/authc/kerberos/KerberosTestCase.java @@ -59,6 +59,8 @@ public abstract class KerberosTestCase extends ESTestCase { /* * Arabic and other language have problems due to handling of generalized time in SimpleKdcServer. For more, look at * org.apache.kerby.asn1.type.Asn1GeneralizedTime#toBytes + * + * Note: several unsupported locales were added in CLDR. #109670 included these below. */ private static Set UNSUPPORTED_LOCALE_LANGUAGES = Set.of( "ar", @@ -81,7 +83,10 @@ public abstract class KerberosTestCase extends ESTestCase { "ur", "pa", "ig", - "sd" + "sd", + "mni", + "sat", + "sa" ); @BeforeClass diff --git a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java index cd37d86626333..e80773d572b03 100644 --- a/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java +++ b/x-pack/qa/oidc-op-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/authc/oidc/OpenIdConnectAuthIT.java @@ -121,6 +121,7 @@ public void testAuthenticateWithCodeFlowAndClientPost() throws Exception { verifyElasticsearchAccessTokenForCodeFlow(tokens.v1()); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/109871") public void testAuthenticateWithCodeFlowAndClientJwtPost() throws Exception { final PrepareAuthResponse prepareAuthResponse = getRedirectedFromFacilitator(REALM_NAME_CLIENT_JWT_AUTH); final String redirectUri = authenticateAtOP(prepareAuthResponse.getAuthUri()); diff --git a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java index 6be2c82e4f3a9..206ccfc432106 100644 --- a/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java +++ b/x-pack/qa/security-tools-tests/src/test/java/org/elasticsearch/xpack/security/authc/file/tool/UsersToolTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.core.PathUtilsForTesting; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.test.FileMatchers; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.authc.support.Hasher; @@ -45,6 +46,7 @@ import static org.elasticsearch.test.SecurityIntegTestCase.getFastStoredHashAlgoForTests; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasSize; public class UsersToolTests extends CommandTestCase { @@ -368,6 +370,30 @@ public void testUseraddNoRoles() throws Exception { assertTrue(lines.toString(), lines.isEmpty()); } + public void testUseraddRolesFileDoesNotExist() throws Exception { + final Path rolesFilePath = confDir.resolve("users_roles"); + Files.delete(rolesFilePath); + var output = execute( + "useradd", + pathHomeParameter, + fileOrderParameter, + "trevor.slattery", + "-p", + SecuritySettingsSourceField.TEST_PASSWORD, + "-r", + "mandarin" + ); + assertThat(output, containsString("does not exist")); + assertThat(output, containsString(rolesFilePath + "]")); + assertThat(output, containsString("attempt to create")); + assertThat(rolesFilePath, FileMatchers.pathExists()); + + List lines = Files.readAllLines(rolesFilePath, StandardCharsets.UTF_8); + assertThat(lines, hasSize(1)); + assertThat(lines.get(0), containsString("trevor.slattery")); + assertThat(lines.get(0), containsString("mandarin")); + } + public void testAddUserWithInvalidHashingAlgorithmInFips() throws Exception { settings = Settings.builder() .put(settings)