-
Notifications
You must be signed in to change notification settings - Fork 25k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add the ability to require an ingest pipeline (#46847)
This commit adds the ability to require an ingest pipeline on an index. Today we can have a default pipeline, but that could be overridden by a request pipeline parameter. This commit introduces a new index setting index.required_pipeline that acts similarly to index.default_pipeline, except that it can not be overridden by a request pipeline parameter. Additionally, a default pipeline and a request pipeline can not both be set. The required pipeline can be set to _none to ensure that no pipeline ever runs for index requests on that index.
- Loading branch information
1 parent
9f65af9
commit 19b710a
Showing
9 changed files
with
511 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
175 changes: 175 additions & 0 deletions
175
modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/240_required_pipeline.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,175 @@ | ||
| --- | ||
| teardown: | ||
| - do: | ||
| ingest.delete_pipeline: | ||
| id: "my_pipeline" | ||
| ignore: 404 | ||
|
|
||
| --- | ||
| "Test index with required pipeline": | ||
| - do: | ||
| ingest.put_pipeline: | ||
| id: "my_pipeline" | ||
| body: > | ||
| { | ||
| "description": "_description", | ||
| "processors": [ | ||
| { | ||
| "bytes" : { | ||
| "field" : "bytes_source_field", | ||
| "target_field" : "bytes_target_field" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| - match: { acknowledged: true } | ||
| # required pipeline via index | ||
| - do: | ||
| indices.create: | ||
| index: test | ||
| body: | ||
| settings: | ||
| index: | ||
| required_pipeline: "my_pipeline" | ||
| aliases: | ||
| test_alias: {} | ||
|
|
||
| - do: | ||
| index: | ||
| index: test | ||
| id: 1 | ||
| body: {bytes_source_field: "1kb"} | ||
|
|
||
| - do: | ||
| get: | ||
| index: test | ||
| id: 1 | ||
| - match: { _source.bytes_source_field: "1kb" } | ||
| - match: { _source.bytes_target_field: 1024 } | ||
| # required pipeline via alias | ||
| - do: | ||
| index: | ||
| index: test_alias | ||
| id: 2 | ||
| body: {bytes_source_field: "1kb"} | ||
|
|
||
| - do: | ||
| get: | ||
| index: test | ||
| id: 2 | ||
| - match: { _source.bytes_source_field: "1kb" } | ||
| - match: { _source.bytes_target_field: 1024 } | ||
| # required pipeline via upsert | ||
| - do: | ||
| update: | ||
| index: test | ||
| id: 3 | ||
| body: | ||
| script: | ||
| source: "ctx._source.ran_script = true" | ||
| lang: "painless" | ||
| upsert: { "bytes_source_field":"1kb" } | ||
| - do: | ||
| get: | ||
| index: test | ||
| id: 3 | ||
| - match: { _source.bytes_source_field: "1kb" } | ||
| - match: { _source.bytes_target_field: 1024 } | ||
| # required pipeline via scripted upsert | ||
| - do: | ||
| update: | ||
| index: test | ||
| id: 4 | ||
| body: | ||
| script: | ||
| source: "ctx._source.bytes_source_field = '1kb'" | ||
| lang: "painless" | ||
| upsert : {} | ||
| scripted_upsert: true | ||
| - do: | ||
| get: | ||
| index: test | ||
| id: 4 | ||
| - match: { _source.bytes_source_field: "1kb" } | ||
| - match: { _source.bytes_target_field: 1024 } | ||
| # required pipeline via doc_as_upsert | ||
| - do: | ||
| update: | ||
| index: test | ||
| id: 5 | ||
| body: | ||
| doc: { "bytes_source_field":"1kb" } | ||
| doc_as_upsert: true | ||
| - do: | ||
| get: | ||
| index: test | ||
| id: 5 | ||
| - match: { _source.bytes_source_field: "1kb" } | ||
| - match: { _source.bytes_target_field: 1024 } | ||
| # required pipeline via bulk upsert | ||
| # note - bulk scripted upsert's execute the pipeline before the script, so any data referenced by the pipeline | ||
| # needs to be in the upsert, not the script | ||
| - do: | ||
| bulk: | ||
| refresh: true | ||
| body: | | ||
| {"update":{"_id":"6","_index":"test"}} | ||
| {"script":"ctx._source.ran_script = true","upsert":{"bytes_source_field":"1kb"}} | ||
| {"update":{"_id":"7","_index":"test"}} | ||
| {"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true} | ||
| {"update":{"_id":"8","_index":"test"}} | ||
| {"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true} | ||
| {"update":{"_id":"6_alias","_index":"test_alias"}} | ||
| {"script":"ctx._source.ran_script = true","upsert":{"bytes_source_field":"1kb"}} | ||
| {"update":{"_id":"7_alias","_index":"test_alias"}} | ||
| {"doc":{"bytes_source_field":"2kb"}, "doc_as_upsert":true} | ||
| {"update":{"_id":"8_alias","_index":"test_alias"}} | ||
| {"script": "ctx._source.ran_script = true","upsert":{"bytes_source_field":"3kb"}, "scripted_upsert" : true} | ||
| - do: | ||
| mget: | ||
| body: | ||
| docs: | ||
| - { _index: "test", _id: "6" } | ||
| - { _index: "test", _id: "7" } | ||
| - { _index: "test", _id: "8" } | ||
| - { _index: "test", _id: "6_alias" } | ||
| - { _index: "test", _id: "7_alias" } | ||
| - { _index: "test", _id: "8_alias" } | ||
| - match: { docs.0._index: "test" } | ||
| - match: { docs.0._id: "6" } | ||
| - match: { docs.0._source.bytes_source_field: "1kb" } | ||
| - match: { docs.0._source.bytes_target_field: 1024 } | ||
| - is_false: docs.0._source.ran_script | ||
| - match: { docs.1._index: "test" } | ||
| - match: { docs.1._id: "7" } | ||
| - match: { docs.1._source.bytes_source_field: "2kb" } | ||
| - match: { docs.1._source.bytes_target_field: 2048 } | ||
| - match: { docs.2._index: "test" } | ||
| - match: { docs.2._id: "8" } | ||
| - match: { docs.2._source.bytes_source_field: "3kb" } | ||
| - match: { docs.2._source.bytes_target_field: 3072 } | ||
| - match: { docs.2._source.ran_script: true } | ||
| - match: { docs.3._index: "test" } | ||
| - match: { docs.3._id: "6_alias" } | ||
| - match: { docs.3._source.bytes_source_field: "1kb" } | ||
| - match: { docs.3._source.bytes_target_field: 1024 } | ||
| - is_false: docs.3._source.ran_script | ||
| - match: { docs.4._index: "test" } | ||
| - match: { docs.4._id: "7_alias" } | ||
| - match: { docs.4._source.bytes_source_field: "2kb" } | ||
| - match: { docs.4._source.bytes_target_field: 2048 } | ||
| - match: { docs.5._index: "test" } | ||
| - match: { docs.5._id: "8_alias" } | ||
| - match: { docs.5._source.bytes_source_field: "3kb" } | ||
| - match: { docs.5._source.bytes_target_field: 3072 } | ||
| - match: { docs.5._source.ran_script: true } | ||
|
|
||
| # bad request, request pipeline can not be specified | ||
| - do: | ||
| catch: /illegal_argument_exception.*request pipeline \[pipeline\] can not override required pipeline \[my_pipeline\]/ | ||
| index: | ||
| index: test | ||
| id: 9 | ||
| pipeline: "pipeline" | ||
| body: {bytes_source_field: "1kb"} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.