From 9d01696dde7a260d1ff73715035ae14008ac70af Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 24 May 2018 09:51:19 +0200 Subject: [PATCH 01/24] Use mappings to format doc-value fields by default. Doc-value fields now return a value that is based on the mappings rather than the script implementation by default. This deprecates the special `use_field_mapping` docvalue format which was added in #29639 only to ease the transition to 7.x and it is not necessary anymore in 7.0. --- .../migration/migrate_7_0/search.asciidoc | 9 +++++ .../search/request/docvalue-fields.asciidoc | 22 +++++----- .../search/request/inner-hits.asciidoc | 5 +-- .../test/search.inner_hits/10_basic.yml | 4 +- .../test/search/10_source_filtering.yml | 21 +++++----- .../rest-api-spec/test/search/30_limits.yml | 9 ++--- .../fetch/subphase/DocValueFieldsContext.java | 2 - .../subphase/DocValueFieldsFetchSubPhase.java | 40 ++++++++----------- .../index/query/InnerHitBuilderTests.java | 7 +--- .../search/fields/SearchFieldsIT.java | 3 +- .../sql/plugin/SqlTranslateResponseTests.java | 3 +- .../execution/search/SqlSourceBuilder.java | 3 +- .../xpack/sql/action/SqlLicenseIT.java | 2 +- .../sql/action/SqlTranslateActionIT.java | 2 +- .../rest-api-spec/test/sql/translate.yml | 7 ++-- .../xpack/qa/sql/nosecurity/CliExplainIT.java | 2 +- 16 files changed, 61 insertions(+), 80 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc index 529bd1fa5995b..795c1dd4c0bab 100644 --- a/docs/reference/migration/migrate_7_0/search.asciidoc +++ b/docs/reference/migration/migrate_7_0/search.asciidoc @@ -78,3 +78,12 @@ for a particular index with the index setting `index.max_regex_length`. Search requests with extra content after the main object will no longer be accepted by the `_search` endpoint. A parsing exception will be thrown instead. + +==== Doc-value fields default format + +The format of doc-value fields is changing to be the same as what could be +obtained in 6.x with the special `use_field_mapping` format. This is mostly a +change for date fields, which are now formatted based on the format that is +configured in the mappings by default. This behavior can be changed by +specifying a <> with the doc-value +field. diff --git a/docs/reference/search/request/docvalue-fields.asciidoc b/docs/reference/search/request/docvalue-fields.asciidoc index 9d917c27ab084..16bc3c399fd49 100644 --- a/docs/reference/search/request/docvalue-fields.asciidoc +++ b/docs/reference/search/request/docvalue-fields.asciidoc @@ -12,9 +12,9 @@ GET /_search "match_all": {} }, "docvalue_fields" : [ + "my_ip_field", <1> { - "field": "my_ip_field", <1> - "format": "use_field_mapping" <2> + "field": "my_keyword_field" <2> }, { "field": "my_date_field", @@ -25,13 +25,10 @@ GET /_search -------------------------------------------------- // CONSOLE <1> the name of the field -<2> the special `use_field_mapping` format tells Elasticsearch to use the format from the mapping -<3> date fields may use a custom format +<2> an object notation is supported as well +<3> the object notation allows to specify a custom format -Doc value fields can work on fields that are not stored. - -Note that if the fields parameter specifies fields without docvalues it will try to load the value from the fielddata cache -causing the terms for that field to be loaded to memory (cached), which will result in more memory consumption. +Doc value fields can work on fields that have doc-values enabled, regardless of whether they are stored. [float] ==== Custom formats @@ -40,9 +37,8 @@ While most fields do not support custom formats, some of them do: - <> fields can take any <>. - <> fields accept a https://docs.oracle.com/javase/8/docs/api/java/text/DecimalFormat.html[DecimalFormat pattern]. -All fields support the special `use_field_mapping` format, which tells -Elasticsearch to use the mappings to figure out a default format. +By default fields are formatted based on a sensible configuration that depends +on their mappings: `long`, `double` and other numeric fields are formatted as +numbers, `keyword` fields are formatted as strings, `date` fields are formatted +with the configured `date` format, etc. -NOTE: The default is currently to return the same output as -<>. However it will change in 7.0 -to behave as if the `use_field_mapping` format was provided. diff --git a/docs/reference/search/request/inner-hits.asciidoc b/docs/reference/search/request/inner-hits.asciidoc index 887ae2bdf149e..d35c61664bac7 100644 --- a/docs/reference/search/request/inner-hits.asciidoc +++ b/docs/reference/search/request/inner-hits.asciidoc @@ -243,10 +243,7 @@ POST test/_search "inner_hits": { "_source" : false, "docvalue_fields" : [ - { - "field": "comments.text.keyword", - "format": "use_field_mapping" - } + "comments.text.keyword" ] } } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml index 884a50507c7b7..22812a90fc0b2 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml @@ -60,7 +60,7 @@ setup: - do: search: - body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ { "field": "_seq_no", "format": "use_field_mapping" } ]} }}, "version": true, "docvalue_fields" : [ { "field": "_seq_no", "format": "use_field_mapping" } ] } + body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ "_seq_no" ] } } }, "version": true, "docvalue_fields" : [ "_seq_no" ] } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } @@ -83,7 +83,7 @@ setup: - do: search: - body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ { "field": "_seq_no", "format": "use_field_mapping" } ]} }}, "version": true, "docvalue_fields" : [ { "field": "_seq_no", "format": "use_field_mapping" } ] } + body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ "_seq_no" ] } } }, "version": true, "docvalue_fields" : [ "_seq_no" ] } - match: { hits.total: 1 } - match: { hits.hits.0._index: "test" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml index 59692873cc456..94fb0f5fbff10 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml @@ -134,12 +134,9 @@ setup: --- "docvalue_fields": - skip: - version: " - 6.3.99" - reason: format option was added in 6.4 - features: warnings + version: " - 6.9.99" + reason: Triggers a deprecation warning before 7.0 - do: - warnings: - - 'Doc-value field [count] is not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with the doc value field in order to opt in for the future behaviour and ease the migration to 7.0.' search: body: docvalue_fields: [ "count" ] @@ -148,12 +145,9 @@ setup: --- "docvalue_fields as url param": - skip: - version: " - 6.3.99" - reason: format option was added in 6.4 - features: warnings + version: " - 6.99.99" + reason: Triggered a deprecation warning before 7.0 - do: - warnings: - - 'Doc-value field [count] is not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with the doc value field in order to opt in for the future behaviour and ease the migration to 7.0.' search: docvalue_fields: [ "count" ] - match: { hits.hits.0.fields.count: [1] } @@ -161,9 +155,12 @@ setup: --- "docvalue_fields with default format": - skip: - version: " - 6.3.99" - reason: format option was added in 6.4 + version: " - 6.99.99" + reason: Only triggers warnings on 7.0+ + features: warnings - do: + warnings: + - "[use_field_mapping] is a special format that was only used to ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore." search: body: docvalue_fields: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml index 905635e1d10b6..215885502749b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml @@ -73,12 +73,9 @@ setup: query: match_all: {} docvalue_fields: - - field: "one" - format: "use_field_mapping" - - field: "two" - format: "use_field_mapping" - - field: "three" - format: "use_field_mapping" + - "one" + - "two" + - "three" --- "Script_fields size limit": diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java index cf1596fd326b9..daafe0970290b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java @@ -38,8 +38,6 @@ */ public class DocValueFieldsContext { - public static final String USE_DEFAULT_FORMAT = "use_field_mapping"; - /** * Wrapper around a field name and the format that should be used to * display values of this field. diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index a1562e118fb86..81f88c5add264 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -28,7 +28,6 @@ import org.elasticsearch.index.fielddata.AtomicNumericFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; @@ -54,6 +53,7 @@ */ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { + private static final String USE_DEFAULT_FORMAT = "use_field_mapping"; private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(DocValueFieldsFetchSubPhase.class)); @Override @@ -64,9 +64,9 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept String name = context.collapse().getFieldType().name(); if (context.docValueFieldsContext() == null) { context.docValueFieldsContext(new DocValueFieldsContext( - Collections.singletonList(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT)))); + Collections.singletonList(new FieldAndFormat(name, null)))); } else if (context.docValueFieldsContext().fields().stream().map(ff -> ff.field).anyMatch(name::equals) == false) { - context.docValueFieldsContext().fields().add(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT)); + context.docValueFieldsContext().fields().add(new FieldAndFormat(name, null)); } } @@ -77,28 +77,21 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept hits = hits.clone(); // don't modify the incoming hits Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId)); + boolean usesDefaultFormat = false; for (FieldAndFormat fieldAndFormat : context.docValueFieldsContext().fields()) { String field = fieldAndFormat.field; MappedFieldType fieldType = context.mapperService().fullName(field); if (fieldType != null) { final IndexFieldData indexFieldData = context.getForField(fieldType); - final DocValueFormat format; - if (fieldAndFormat.format == null) { - DEPRECATION_LOGGER.deprecated("Doc-value field [" + fieldAndFormat.field + "] is not using a format. The output will " + - "change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass " + - "[format={}] with the doc value field in order to opt in for the future behaviour and ease the migration to " + - "7.0.", DocValueFieldsContext.USE_DEFAULT_FORMAT); - format = null; - } else { - String formatDesc = fieldAndFormat.format; - if (Objects.equals(formatDesc, DocValueFieldsContext.USE_DEFAULT_FORMAT)) { - formatDesc = null; - } - format = fieldType.docValueFormat(formatDesc, null); + String formatDesc = fieldAndFormat.format; + if (Objects.equals(formatDesc, USE_DEFAULT_FORMAT)) { + // TODO: Remove in 8.x + usesDefaultFormat = true; + formatDesc = null; } + final DocValueFormat format = fieldType.docValueFormat(formatDesc, null); LeafReaderContext subReaderContext = null; AtomicFieldData data = null; - ScriptDocValues scriptValues = null; // legacy SortedBinaryDocValues binaryValues = null; // binary / string / ip fields SortedNumericDocValues longValues = null; // int / date fields SortedNumericDoubleValues doubleValues = null; // floating-point fields @@ -108,9 +101,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept int readerIndex = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves()); subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); data = indexFieldData.load(subReaderContext); - if (format == null) { - scriptValues = data.getScriptValues(); - } else if (indexFieldData instanceof IndexNumericFieldData) { + if (indexFieldData instanceof IndexNumericFieldData) { if (((IndexNumericFieldData) indexFieldData).getNumericType().isFloatingPoint()) { doubleValues = ((AtomicNumericFieldData) data).getDoubleValues(); } else { @@ -131,10 +122,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept final List values = hitField.getValues(); int subDocId = hit.docId() - subReaderContext.docBase; - if (scriptValues != null) { - scriptValues.setNextDocId(subDocId); - values.addAll(scriptValues); - } else if (binaryValues != null) { + if (binaryValues != null) { if (binaryValues.advanceExact(subDocId)) { for (int i = 0, count = binaryValues.docValueCount(); i < count; ++i) { values.add(format.format(binaryValues.nextValue())); @@ -158,5 +146,9 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept } } } + if (usesDefaultFormat) { + DEPRECATION_LOGGER.deprecated("[" + USE_DEFAULT_FORMAT + "] is a special format that was only used to " + + "ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore."); + } } } diff --git a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java index a2068a666f44c..3f28819f338fb 100644 --- a/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.FieldAndFormat; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests; @@ -150,8 +149,7 @@ public static InnerHitBuilder randomInnerHits() { innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16))); } innerHits.setDocValueFields(randomListStuff(16, - () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), - randomBoolean() ? null : DocValueFieldsContext.USE_DEFAULT_FORMAT))); + () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null))); // Random script fields deduped on their field name. Map scriptFields = new HashMap<>(); for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) { @@ -192,8 +190,7 @@ static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException { modifiers.add(() -> { if (randomBoolean()) { copy.setDocValueFields(randomValueOtherThan(copy.getDocValueFields(), - () -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), - randomBoolean() ? null : DocValueFieldsContext.USE_DEFAULT_FORMAT)))); + () -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null)))); } else { copy.addDocValueField(randomAlphaOfLengthBetween(1, 16)); } diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index a8a2669ef9b4a..aad3714ff5589 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -798,7 +798,8 @@ public void testFieldsPulledFromFieldData() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), equalTo(date)); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), + equalTo(Joda.forPattern("dateOptionalTime").printer().print(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); diff --git a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlTranslateResponseTests.java b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlTranslateResponseTests.java index 76f73fada0663..061b70a55d975 100644 --- a/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlTranslateResponseTests.java +++ b/x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlTranslateResponseTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.test.AbstractStreamableTestCase; import java.io.IOException; @@ -19,7 +18,7 @@ protected SqlTranslateResponse createTestInstance() { if (randomBoolean()) { long docValues = iterations(5, 10); for (int i = 0; i < docValues; i++) { - s.docValueField(randomAlphaOfLength(10), DocValueFieldsContext.USE_DEFAULT_FORMAT); + s.docValueField(randomAlphaOfLength(10)); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java index 4c4bee9440ba1..8fecd2c042671 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java @@ -69,8 +69,7 @@ public void build(SearchSourceBuilder sourceBuilder) { if (!sourceFields.isEmpty()) { sourceBuilder.fetchSource(sourceFields.toArray(Strings.EMPTY_ARRAY), null); } - docFields.forEach(field -> sourceBuilder.docValueField(field.field, - field.format == null ? DocValueFieldsContext.USE_DEFAULT_FORMAT : field.format)); + docFields.forEach(field -> sourceBuilder.docValueField(field.field, field.format)); scriptFields.forEach(sourceBuilder::scriptField); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java index 8988f70672ac6..431aeb3e496bd 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java @@ -152,7 +152,7 @@ public void testSqlTranslateActionLicense() throws Exception { SqlTranslateResponse response = client().prepareExecute(SqlTranslateAction.INSTANCE).query("SELECT * FROM test").get(); SearchSourceBuilder source = response.source(); assertThat(source.docValueFields(), Matchers.contains( - new DocValueFieldsContext.FieldAndFormat("count", DocValueFieldsContext.USE_DEFAULT_FORMAT))); + new DocValueFieldsContext.FieldAndFormat("count", null))); FetchSourceContext fetchSource = source.fetchSource(); assertThat(fetchSource.includes(), Matchers.arrayContaining("data")); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java index a4c440eb9dfad..71b3e8ba881fb 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlTranslateActionIT.java @@ -37,7 +37,7 @@ public void testSqlTranslateAction() throws Exception { assertEquals(true, fetch.fetchSource()); assertArrayEquals(new String[] { "data" }, fetch.includes()); assertEquals( - singletonList(new DocValueFieldsContext.FieldAndFormat("count", DocValueFieldsContext.USE_DEFAULT_FORMAT)), + singletonList(new DocValueFieldsContext.FieldAndFormat("count", null)), source.docValueFields()); assertEquals(singletonList(SortBuilders.fieldSort("count")), source.sorts()); } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml index f47ea2d4d7e15..b791395c2baa2 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml @@ -1,9 +1,8 @@ --- "Translate SQL": - skip: - version: " - 6.3.99" - reason: format option was added in 6.4 - features: warnings + version: " - 6.99.99" + reason: Triggers warnings before 7.0 - do: bulk: @@ -29,7 +28,7 @@ excludes: [] docvalue_fields: - field: int - format: use_field_mapping + format: null sort: - int: order: asc diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java b/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java index 63831c2d4dec0..9dc1972488e5b 100644 --- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java +++ b/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java @@ -105,7 +105,7 @@ public void testExplainWithWhere() throws IOException { assertThat(readLine(), startsWith(" \"docvalue_fields\" : [")); assertThat(readLine(), startsWith(" {")); assertThat(readLine(), startsWith(" \"field\" : \"i\"")); - assertThat(readLine(), startsWith(" \"format\" : \"use_field_mapping\"")); + assertThat(readLine(), startsWith(" \"format\" : null)); assertThat(readLine(), startsWith(" }")); assertThat(readLine(), startsWith(" ],")); assertThat(readLine(), startsWith(" \"sort\" : [")); From 71401497aad1cf58ce4606ad9b1eb428578f0621 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 24 May 2018 15:50:09 +0200 Subject: [PATCH 02/24] iter --- .../rest-api-spec/test/search.inner_hits/10_basic.yml | 4 ++-- .../main/resources/rest-api-spec/test/search/30_limits.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml index 22812a90fc0b2..c74ec861d9af1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml @@ -45,8 +45,8 @@ setup: "Nested doc version and seqIDs": - skip: - version: " - 6.3.99" - reason: "object notation for docvalue_fields was introduced in 6.4" + version: " - 6.99.99" + reason: "Triggers warnings before 7.0" - do: index: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml index 215885502749b..f9787e7f230ca 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml @@ -63,8 +63,8 @@ setup: "Docvalues_fields size limit": - skip: - version: " - 6.3.99" - reason: "The object notation for docvalue_fields is only supported on 6.4+" + version: " - 6.99.99" + reason: "Triggers warnings before 7.0" - do: catch: /Trying to retrieve too many docvalue_fields\. Must be less than or equal to[:] \[2\] but was \[3\]\. This limit can be set by changing the \[index.max_docvalue_fields_search\] index level setting\./ search: From b062fd6cb55348d01d80ac5edc57ff9880ab0785 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 5 Jun 2018 15:48:41 +0200 Subject: [PATCH 03/24] iter --- .../java/org/elasticsearch/search/fields/SearchFieldsIT.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index a6628bc91e466..3f9e586c99fe6 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -807,8 +807,7 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); - assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), - equalTo(new BytesRef(new byte[] {42, 100}))); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ")); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); builder = client().prepareSearch().setQuery(matchAllQuery()) From 92696e9cc88537a3cef22a7e5a23de4694027211 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 6 Jun 2018 11:38:35 +0200 Subject: [PATCH 04/24] iter --- .../src/test/resources/rest-api-spec/test/sql/translate.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml index b791395c2baa2..1c936115e51d4 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/sql/translate.yml @@ -28,7 +28,6 @@ excludes: [] docvalue_fields: - field: int - format: null sort: - int: order: asc From ec5633f68db2681dd44c56c75e986193badce51b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 20 Jun 2018 10:17:36 +0200 Subject: [PATCH 05/24] iter --- .../java/org/elasticsearch/search/fields/SearchFieldsIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 3f9e586c99fe6..36dcfb66949b9 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.fields; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; From 519517b316b1bd77ab3e4cefab3fdf53bceef83b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 22 Jun 2018 14:08:33 +0200 Subject: [PATCH 06/24] iter --- .../xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java index 57681a0aafbb2..4a9649e537d7d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java @@ -120,7 +120,7 @@ private SearchRequestBuilder buildSearchRequest(long start) { if (docValueField.equals(context.extractedFields.timeField())) { searchRequestBuilder.addDocValueField(docValueField, EPOCH_MILLIS_FORMAT); } else { - searchRequestBuilder.addDocValueField(docValueField, DocValueFieldsContext.USE_DEFAULT_FORMAT); + searchRequestBuilder.addDocValueField(docValueField); } } String[] sourceFields = context.extractedFields.getSourceFields(); From 20fd9181bafd8600563ea21b9f6a92f5940b9eb9 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 25 Jun 2018 15:32:02 +0200 Subject: [PATCH 07/24] iter --- docs/reference/migration/migrate_7_0/search.asciidoc | 2 +- x-pack/docs/en/sql/endpoints/translate.asciidoc | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/reference/migration/migrate_7_0/search.asciidoc b/docs/reference/migration/migrate_7_0/search.asciidoc index e8df255063e69..e9fd50e819d3c 100644 --- a/docs/reference/migration/migrate_7_0/search.asciidoc +++ b/docs/reference/migration/migrate_7_0/search.asciidoc @@ -91,7 +91,7 @@ The format of doc-value fields is changing to be the same as what could be obtained in 6.x with the special `use_field_mapping` format. This is mostly a change for date fields, which are now formatted based on the format that is configured in the mappings by default. This behavior can be changed by -specifying a <> with the doc-value +specifying a <> within the doc-value field. ==== Semantics changed for `max_concurrent_shard_requests` diff --git a/x-pack/docs/en/sql/endpoints/translate.asciidoc b/x-pack/docs/en/sql/endpoints/translate.asciidoc index be6a77a3caa44..582e9625115be 100644 --- a/x-pack/docs/en/sql/endpoints/translate.asciidoc +++ b/x-pack/docs/en/sql/endpoints/translate.asciidoc @@ -23,8 +23,7 @@ Which returns: "size" : 10, "docvalue_fields" : [ { - "field": "page_count", - "format": "use_field_mapping" + "field": "page_count" }, { "field": "release_date", From 5fdd429998fa50c11066aabb975449de2a804b39 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 25 Jun 2018 19:03:13 +0200 Subject: [PATCH 08/24] iter --- .../xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java index 4a9649e537d7d..99d1cec2aa7e7 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java @@ -20,7 +20,6 @@ import org.elasticsearch.script.Script; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.StoredFieldsContext; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.datafeed.extractor.DataExtractor; From 635e0a11568e9f579de977da5e9dbc528aa498c9 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 27 Jun 2018 14:25:50 +0200 Subject: [PATCH 09/24] iter --- .../xpack/sql/execution/search/SqlSourceBuilder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java index 8fecd2c042671..1872748c328fa 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java @@ -8,7 +8,6 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.script.Script; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.FieldAndFormat; import java.util.LinkedHashMap; From 20e7f1394b4b5eedc08367ace604c6ae8bdeae44 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Sep 2018 12:06:10 +0200 Subject: [PATCH 10/24] iter --- .../search/fields/SearchFieldsIT.java | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 73cdce351f643..90a8b6aeb872d 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -800,8 +800,8 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - ZonedDateTime dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(); - assertThat(dateField.toInstant().toEpochMilli(), equalTo(date.toInstant().toEpochMilli())); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), + equalTo(DateFormatters.forPattern("dateOptionalTime").format(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); @@ -825,13 +825,12 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L)); assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); - dateField = searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(); - assertThat(dateField.toInstant().toEpochMilli(), equalTo(date.toInstant().toEpochMilli())); + assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), + equalTo(DateFormatters.forPattern("dateOptionalTime").format(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); - assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), - equalTo(new BytesRef(new byte[] {42, 100}))); + assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ")); assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1")); builder = client().prepareSearch().setQuery(matchAllQuery()) @@ -996,9 +995,7 @@ public void testDocValueFieldsWithFieldAlias() throws Exception { DocumentField dateField = fields.get("date_field"); assertThat(dateField.getName(), equalTo("date_field")); - - ZonedDateTime fetchedDate = dateField.getValue(); - assertThat(fetchedDate, equalTo(date)); + assertThat(dateField.getValue(), equalTo("1990-12-29")); } public void testWildcardDocValueFieldsWithFieldAlias() throws Exception { @@ -1061,8 +1058,7 @@ public void testWildcardDocValueFieldsWithFieldAlias() throws Exception { DocumentField dateField = fields.get("date_field"); assertThat(dateField.getName(), equalTo("date_field")); - ZonedDateTime fetchedDate = dateField.getValue(); - assertThat(fetchedDate, equalTo(date)); + assertThat(dateField.getValue(), equalTo("1990-12-29")); } From 6e57e4751d20aa06e0fa597450d412b61c920152 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 13 Sep 2018 12:10:34 +0200 Subject: [PATCH 11/24] iter --- .../subphase/DocValueFieldsFetchSubPhase.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index aff28718c3da9..d981295d8459f 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -18,12 +18,12 @@ */ package org.elasticsearch.search.fetch.subphase; +import org.apache.logging.log4j.LogManager; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.index.fielddata.AtomicFieldData; import org.elasticsearch.index.fielddata.AtomicNumericFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -54,7 +54,7 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { private static final String USE_DEFAULT_FORMAT = "use_field_mapping"; - private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(DocValueFieldsFetchSubPhase.class)); + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(DocValueFieldsFetchSubPhase.class)); @Override public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException { @@ -77,7 +77,15 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept hits = hits.clone(); // don't modify the incoming hits Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId)); - boolean usesDefaultFormat = false; + if (context.docValueFieldsContext().fields().stream() + .map(f -> f.format) + .filter(USE_DEFAULT_FORMAT::equals) + .findAny() + .isPresent()) { + DEPRECATION_LOGGER.deprecated("[" + USE_DEFAULT_FORMAT + "] is a special format that was only used to " + + "ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore."); + } + for (FieldAndFormat fieldAndFormat : context.docValueFieldsContext().fields()) { String field = fieldAndFormat.field; MappedFieldType fieldType = context.mapperService().fullName(field); @@ -86,7 +94,6 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept String formatDesc = fieldAndFormat.format; if (Objects.equals(formatDesc, USE_DEFAULT_FORMAT)) { // TODO: Remove in 8.x - usesDefaultFormat = true; formatDesc = null; } final DocValueFormat format = fieldType.docValueFormat(formatDesc, null); @@ -146,9 +153,5 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept } } } - if (usesDefaultFormat) { - DEPRECATION_LOGGER.deprecated("[" + USE_DEFAULT_FORMAT + "] is a special format that was only used to " + - "ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore."); - } } } From cbbca3ed508f5754a1366e9c99291c963ea4a7d2 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 14 Sep 2018 14:26:21 +0200 Subject: [PATCH 12/24] iter --- .../search/fetch/subphase/DocValueFieldsFetchSubPhase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index d981295d8459f..e94ac0fdf6c99 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -54,7 +54,8 @@ public final class DocValueFieldsFetchSubPhase implements FetchSubPhase { private static final String USE_DEFAULT_FORMAT = "use_field_mapping"; - private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(DocValueFieldsFetchSubPhase.class)); + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger( + LogManager.getLogger(DocValueFieldsFetchSubPhase.class)); @Override public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOException { From 29563520d37c1d596200d4253ec05c1d0193ab71 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 18 Sep 2018 12:38:37 +0200 Subject: [PATCH 13/24] iter --- .../rest-api-spec/test/search/10_source_filtering.yml | 7 ++----- .../search/fetch/subphase/DocValueFieldsFetchSubPhase.java | 1 - .../org/elasticsearch/search/fields/SearchFieldsIT.java | 1 - 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml index 7ac5e8f9ce7d4..5d2b8ddf013f9 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml @@ -145,12 +145,9 @@ setup: --- "multiple docvalue_fields": - skip: - version: " - 6.4.0" - reason: format option was added in 6.4 and the deprecation message changed in 6.4.1 - features: warnings + version: " - 6.9.99" + reason: Triggered a deprecation warning before 7.0 - do: - warnings: - - 'There are doc-value fields which are not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with a doc value field in order to opt in for the future behaviour and ease the migration to 7.0: [count, include.field1.keyword]' search: body: docvalue_fields: [ "count", "include.field1.keyword" ] diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java index be6a7bbec9360..e94ac0fdf6c99 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java @@ -45,7 +45,6 @@ import java.util.HashMap; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; /** * Query sub phase which pulls data from doc values diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index 34204f097f98b..c068b6b1ca37b 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -48,7 +48,6 @@ import org.elasticsearch.test.InternalSettingsPlugin; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.joda.time.ReadableDateTime; import org.joda.time.format.DateTimeFormat; import java.time.ZoneOffset; From 9b96e104f302763dea27287a2fd97f8810541da0 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 25 Sep 2018 14:25:28 +0200 Subject: [PATCH 14/24] iter --- .../org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java b/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java index 7a98ded8255af..567f3470e2049 100644 --- a/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java +++ b/x-pack/qa/sql/no-security/src/test/java/org/elasticsearch/xpack/qa/sql/nosecurity/CliExplainIT.java @@ -105,7 +105,7 @@ public void testExplainWithWhere() throws IOException { assertThat(readLine(), startsWith(" \"docvalue_fields\" : [")); assertThat(readLine(), startsWith(" {")); assertThat(readLine(), startsWith(" \"field\" : \"i\"")); - assertThat(readLine(), startsWith(" \"format\" : null)); + assertThat(readLine(), startsWith(" \"format\" : null")); assertThat(readLine(), startsWith(" }")); assertThat(readLine(), startsWith(" ],")); assertThat(readLine(), startsWith(" \"sort\" : [")); From 091d195590eacf1f1a50b263288539ead2af9276 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 26 Nov 2018 18:46:04 +0100 Subject: [PATCH 15/24] iter --- .../xpack/ml/datafeed/extractor/fields/ExtractedField.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java index 232cd53a359ce..8ddeb966f7c73 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java @@ -53,7 +53,7 @@ public ExtractionMethod getExtractionMethod() { public abstract Object[] value(SearchHit hit); public String getDocValueFormat() { - return DocValueFieldsContext.USE_DEFAULT_FORMAT; + return null; } public static ExtractedField newTimeField(String name, ExtractionMethod extractionMethod) { From 30922bfc4e3b3d755dde4976bc8a691a14ccc759 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 26 Nov 2018 19:08:33 +0100 Subject: [PATCH 16/24] iter --- .../xpack/ml/datafeed/extractor/fields/ExtractedField.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java index 8ddeb966f7c73..4011f9493f916 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedField.java @@ -7,7 +7,6 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.joda.time.base.BaseDateTime; import java.util.List; From 854951f7061c2e9af848d25db74758b2accaa6ee Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 27 Nov 2018 11:06:53 +0100 Subject: [PATCH 17/24] iter --- .../ml/datafeed/extractor/fields/ExtractedFieldTests.java | 3 +-- .../ml/datafeed/extractor/fields/ExtractedFieldsTests.java | 3 +-- .../extractor/fields/TimeBasedExtractedFieldsTests.java | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java index 1e5e6fa652db1..1b9e5148e697b 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.ml.datafeed.extractor.fields; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.ml.datafeed.extractor.fields.ExtractedField; import org.elasticsearch.xpack.ml.test.SearchHitBuilder; @@ -145,7 +144,7 @@ public void testAliasVersusName() { public void testGetDocValueFormat() { for (ExtractedField.ExtractionMethod method : ExtractedField.ExtractionMethod.values()) { - assertThat(ExtractedField.newField("f", method).getDocValueFormat(), equalTo(DocValueFieldsContext.USE_DEFAULT_FORMAT)); + assertThat(ExtractedField.newField("f", method).getDocValueFormat(), equalTo(null)); } assertThat(ExtractedField.newTimeField("doc_value_time", ExtractedField.ExtractionMethod.DOC_VALUE).getDocValueFormat(), equalTo("epoch_millis")); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldsTests.java index 222531141364a..db25f820dbbd8 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldsTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/ExtractedFieldsTests.java @@ -7,7 +7,6 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; @@ -63,7 +62,7 @@ public void testBuildGivenMixtureOfTypes() { assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("time")); assertThat(extractedFields.getDocValueFields().get(0).getDocValueFormat(), equalTo("epoch_millis")); assertThat(extractedFields.getDocValueFields().get(1).getName(), equalTo("value")); - assertThat(extractedFields.getDocValueFields().get(1).getDocValueFormat(), equalTo(DocValueFieldsContext.USE_DEFAULT_FORMAT)); + assertThat(extractedFields.getDocValueFields().get(1).getDocValueFormat(), equalTo(null)); assertThat(extractedFields.getSourceFields(), equalTo(new String[] {"airline"})); assertThat(extractedFields.getAllFields().size(), equalTo(4)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFieldsTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFieldsTests.java index 5e388afad282a..8a6814f75115c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFieldsTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/fields/TimeBasedExtractedFieldsTests.java @@ -9,7 +9,6 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; @@ -143,7 +142,7 @@ public void testBuildGivenMixtureOfTypes() { assertThat(extractedFields.getDocValueFields().get(0).getName(), equalTo("time")); assertThat(extractedFields.getDocValueFields().get(0).getDocValueFormat(), equalTo("epoch_millis")); assertThat(extractedFields.getDocValueFields().get(1).getName(), equalTo("value")); - assertThat(extractedFields.getDocValueFields().get(1).getDocValueFormat(), equalTo(DocValueFieldsContext.USE_DEFAULT_FORMAT)); + assertThat(extractedFields.getDocValueFields().get(1).getDocValueFormat(), equalTo(null)); assertThat(extractedFields.getSourceFields().length, equalTo(1)); assertThat(extractedFields.getSourceFields()[0], equalTo("airline")); assertThat(extractedFields.getAllFields().size(), equalTo(4)); From 48aacd678c13f75e027ea6a567a19164480e9210 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 28 Nov 2018 13:04:56 +0100 Subject: [PATCH 18/24] iter --- .../test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java | 2 +- .../elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java index c3cd8f61538fe..20fff9e4da83e 100644 --- a/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java +++ b/qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/QueryBuilderBWCIT.java @@ -196,7 +196,7 @@ public void testQueryBuilderBWC() throws Exception { QueryBuilder expectedQueryBuilder = (QueryBuilder) CANDIDATES.get(i)[1]; Request request = new Request("GET", "/" + index + "/_search"); request.setJsonEntity("{\"query\": {\"ids\": {\"values\": [\"" + Integer.toString(i) + "\"]}}, " + - "\"docvalue_fields\": [{\"field\":\"query.query_builder_field\", \"format\":\"use_field_mapping\"}]}"); + "\"docvalue_fields\": [{\"field\":\"query.query_builder_field\"}]}"); Response rsp = client().performRequest(request); assertEquals(200, rsp.getStatusLine().getStatusCode()); Map hitRsp = (Map) ((List) ((Map)toMap(rsp).get("hits")).get("hits")).get(0); diff --git a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java index f47e1f013d7fb..a8f850ea9e245 100644 --- a/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java +++ b/x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/CliExplainIT.java @@ -105,7 +105,6 @@ public void testExplainWithWhere() throws IOException { assertThat(readLine(), startsWith(" \"docvalue_fields\" : [")); assertThat(readLine(), startsWith(" {")); assertThat(readLine(), startsWith(" \"field\" : \"i\"")); - assertThat(readLine(), startsWith(" \"format\" : null")); assertThat(readLine(), startsWith(" }")); assertThat(readLine(), startsWith(" ],")); assertThat(readLine(), startsWith(" \"sort\" : [")); From 8f2329f5b3742441a50adbf164565bf8193c5662 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Dec 2018 10:14:05 +0100 Subject: [PATCH 19/24] iter --- .../xpack/sql/querydsl/container/QueryContainer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java index 8901ca75c8f88..8857ffcf858fe 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainer.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.execution.search.FieldExtraction; import org.elasticsearch.xpack.sql.execution.search.SourceGenerator; @@ -183,7 +182,7 @@ private Tuple nestedHitFieldRef(FieldAttribute List nestedRefs = new ArrayList<>(); String name = aliasName(attr); - String format = attr.field().getDataType() == DataType.DATE ? "epoch_millis" : DocValueFieldsContext.USE_DEFAULT_FORMAT; + String format = attr.field().getDataType() == DataType.DATE ? "epoch_millis" : null; Query q = rewriteToContainNestedField(query, attr.location(), attr.nestedParent().name(), name, format, attr.field().isAggregatable()); From da84b026dfa03bdb924530de8e283f800bf7521c Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 6 Dec 2018 12:03:07 +0100 Subject: [PATCH 20/24] iter --- .../querydsl/container/QueryContainerTests.java | 3 +-- .../xpack/sql/querydsl/query/BoolQueryTests.java | 8 +++----- .../xpack/sql/querydsl/query/LeafQueryTests.java | 4 +--- .../sql/querydsl/query/NestedQueryTests.java | 16 +++++++--------- 4 files changed, 12 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java index 84cb62a5ada06..add35f9a449a7 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/container/QueryContainerTests.java @@ -7,7 +7,6 @@ import java.util.AbstractMap.SimpleImmutableEntry; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.querydsl.query.BoolQuery; import org.elasticsearch.xpack.sql.querydsl.query.MatchAll; @@ -24,7 +23,7 @@ public class QueryContainerTests extends ESTestCase { private Location location = LocationTests.randomLocation(); private String path = randomAlphaOfLength(5); private String name = randomAlphaOfLength(5); - private String format = DocValueFieldsContext.USE_DEFAULT_FORMAT; + private String format = null; private boolean hasDocValues = randomBoolean(); public void testRewriteToContainNestedFieldNoQuery() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java index 5837b069654ac..d2764333a5001 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/BoolQueryTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.querydsl.query; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.tree.Location; @@ -52,15 +51,14 @@ public void testContainsNestedField() { public void testAddNestedField() { Query q = boolQueryWithoutNestedChildren(); - assertSame(q, q.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, - randomBoolean())); + assertSame(q, q.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean())); String path = randomAlphaOfLength(5); String field = randomAlphaOfLength(5); q = boolQueryWithNestedChildren(path, field); String newField = randomAlphaOfLength(5); boolean hasDocValues = randomBoolean(); - Query rewritten = q.addNestedField(path, newField, DocValueFieldsContext.USE_DEFAULT_FORMAT, hasDocValues); + Query rewritten = q.addNestedField(path, newField, null, hasDocValues); assertNotSame(q, rewritten); assertTrue(rewritten.containsNestedField(path, newField)); } @@ -86,7 +84,7 @@ private Query boolQueryWithoutNestedChildren() { private Query boolQueryWithNestedChildren(String path, String field) { NestedQuery match = new NestedQuery(LocationTests.randomLocation(), path, - singletonMap(field, new SimpleImmutableEntry<>(randomBoolean(), DocValueFieldsContext.USE_DEFAULT_FORMAT)), + singletonMap(field, new SimpleImmutableEntry<>(randomBoolean(), null)), new MatchAll(LocationTests.randomLocation())); Query matchAll = new MatchAll(LocationTests.randomLocation()); Query left; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java index 52043003ab5f3..d4b1c33761007 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/LeafQueryTests.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.sql.querydsl.query; import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.tree.Location; @@ -53,8 +52,7 @@ public void testContainsNestedField() { public void testAddNestedField() { Query query = new DummyLeafQuery(LocationTests.randomLocation()); // Leaf queries don't contain nested fields. - assertSame(query, query.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, - randomBoolean())); + assertSame(query, query.addNestedField(randomAlphaOfLength(5), randomAlphaOfLength(5), null, randomBoolean())); } public void testEnrichNestedSort() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java index 3e22895494968..e5288f3f2bf2d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.querydsl.query; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; @@ -44,7 +43,7 @@ private static Map> randomFields() { int size = between(0, 5); Map> fields = new HashMap<>(size); while (fields.size() < size) { - fields.put(randomAlphaOfLength(5), new SimpleImmutableEntry<>(randomBoolean(), DocValueFieldsContext.USE_DEFAULT_FORMAT)); + fields.put(randomAlphaOfLength(5), new SimpleImmutableEntry<>(randomBoolean(), null)); } return fields; } @@ -79,18 +78,18 @@ public void testAddNestedField() { NestedQuery q = randomNestedQuery(0); for (String field : q.fields().keySet()) { // add does nothing if the field is already there - assertSame(q, q.addNestedField(q.path(), field, DocValueFieldsContext.USE_DEFAULT_FORMAT, randomBoolean())); + assertSame(q, q.addNestedField(q.path(), field, null, randomBoolean())); String otherPath = randomValueOtherThan(q.path(), () -> randomAlphaOfLength(5)); // add does nothing if the path doesn't match - assertSame(q, q.addNestedField(otherPath, randomAlphaOfLength(5), DocValueFieldsContext.USE_DEFAULT_FORMAT, randomBoolean())); + assertSame(q, q.addNestedField(otherPath, randomAlphaOfLength(5), null, randomBoolean())); } // if the field isn't in the list then add rewrites to a query with all the old fields and the new one String newField = randomValueOtherThanMany(q.fields()::containsKey, () -> randomAlphaOfLength(5)); boolean hasDocValues = randomBoolean(); - NestedQuery added = (NestedQuery) q.addNestedField(q.path(), newField, DocValueFieldsContext.USE_DEFAULT_FORMAT, hasDocValues); + NestedQuery added = (NestedQuery) q.addNestedField(q.path(), newField, null, hasDocValues); assertNotSame(q, added); - assertThat(added.fields(), hasEntry(newField, new SimpleImmutableEntry<>(hasDocValues, DocValueFieldsContext.USE_DEFAULT_FORMAT))); + assertThat(added.fields(), hasEntry(newField, new SimpleImmutableEntry<>(hasDocValues, null))); assertTrue(added.containsNestedField(q.path(), newField)); for (Map.Entry> field : q.fields().entrySet()) { assertThat(added.fields(), hasEntry(field.getKey(), field.getValue())); @@ -132,8 +131,7 @@ public void testEnrichNestedSort() { public void testToString() { NestedQuery q = new NestedQuery(new Location(1, 1), "a.b", - singletonMap("f", new SimpleImmutableEntry<>(true, DocValueFieldsContext.USE_DEFAULT_FORMAT)), - new MatchAll(new Location(1, 1))); - assertEquals("NestedQuery@1:2[a.b.{f=true=use_field_mapping}[MatchAll@1:2[]]]", q.toString()); + singletonMap("f", new SimpleImmutableEntry<>(true, null)), new MatchAll(new Location(1, 1))); + assertEquals("NestedQuery@1:2[a.b.{f=true=null}[MatchAll@1:2[]]]", q.toString()); } } From 6da69205045831b5875f990e5c781360866ee000 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 14 Jan 2019 19:58:17 +0100 Subject: [PATCH 21/24] iter --- .../datafeed/persistence/DatafeedConfigProvider.java | 4 ++-- .../xpack/ml/job/persistence/JobConfigProvider.java | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index d9ea6cb7c32e4..cd615a61a9a16 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -198,7 +198,7 @@ public void onFailure(Exception e) { public void findDatafeedsForJobIds(Collection jobIds, ActionListener> listener) { SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedJobIdsQuery(jobIds)); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -366,7 +366,7 @@ public void expandDatafeedIds(String expression, boolean allowNoDatafeeds, Actio SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildDatafeedIdQuery(tokens)); sourceBuilder.sort(DatafeedConfig.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(DatafeedConfig.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 73b1fe155fc3c..8e14774c4e07a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -472,7 +472,7 @@ public void jobIdMatches(List ids, ActionListener> listener SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(boolQueryBuilder); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -557,8 +557,8 @@ public void expandJobsIds(String expression, boolean allowNoJobs, boolean exclud SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); - sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); + sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), null); SearchRequest searchRequest = client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -602,8 +602,8 @@ private SearchRequest makeExpandIdsSearchRequest(String expression, boolean excl SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(buildQuery(tokens, excludeDeleting)); sourceBuilder.sort(Job.ID.getPreferredName()); sourceBuilder.fetchSource(false); - sourceBuilder.docValueField(Job.ID.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); - sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), DocValueFieldsContext.USE_DEFAULT_FORMAT); + sourceBuilder.docValueField(Job.ID.getPreferredName(), null); + sourceBuilder.docValueField(Job.GROUPS.getPreferredName(), null); return client.prepareSearch(AnomalyDetectorsIndex.configIndexName()) .setIndicesOptions(IndicesOptions.lenientExpandOpen()) @@ -686,7 +686,7 @@ public void expandGroupIds(List groupIds, ActionListener Date: Tue, 15 Jan 2019 10:15:40 +0100 Subject: [PATCH 22/24] iter --- .../xpack/ml/datafeed/persistence/DatafeedConfigProvider.java | 1 - .../xpack/ml/job/persistence/JobConfigProvider.java | 1 - 2 files changed, 2 deletions(-) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java index cd615a61a9a16..36e71de8bcba1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/persistence/DatafeedConfigProvider.java @@ -44,7 +44,6 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedUpdate; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java index 8e14774c4e07a..eaf51270725ff 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/persistence/JobConfigProvider.java @@ -55,7 +55,6 @@ import org.elasticsearch.index.query.WildcardQueryBuilder; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; From c49e6210b17f1ac02bd0e9f3ab7d81b9d3adb1c8 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Thu, 24 Jan 2019 21:18:39 +0100 Subject: [PATCH 23/24] iter --- .../java/org/elasticsearch/search/fields/SearchFieldsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java index a82d7d710edd9..81315071273e0 100644 --- a/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java +++ b/server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java @@ -803,7 +803,7 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), - equalTo(DateFormatters.forPattern("dateOptionalTime").format(date))); + equalTo(DateFormatter.forPattern("dateOptionalTime").format(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); @@ -828,7 +828,7 @@ public void testDocValueFields() throws Exception { assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0)); assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d)); assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), - equalTo(DateFormatters.forPattern("dateOptionalTime").format(date))); + equalTo(DateFormatter.forPattern("dateOptionalTime").format(date))); assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true)); assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo")); assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo")); From 60cb3e47c2cc8caed2a07acd432d412aca4f6c6b Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Fri, 25 Jan 2019 06:58:49 +0100 Subject: [PATCH 24/24] fix test failure --- .../xpack/sql/querydsl/query/NestedQueryTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java index d28e96644cdf9..818ba04fa1882 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/querydsl/query/NestedQueryTests.java @@ -134,6 +134,6 @@ public void testToString() { NestedQuery q = new NestedQuery(new Source(1, 1, StringUtils.EMPTY), "a.b", singletonMap("f", new SimpleImmutableEntry<>(true, null)), new MatchAll(new Source(1, 1, StringUtils.EMPTY))); - assertEquals("NestedQuery@1:2[a.b.{f=true=use_field_mapping}[MatchAll@1:2[]]]", q.toString()); + assertEquals("NestedQuery@1:2[a.b.{f=true=null}[MatchAll@1:2[]]]", q.toString()); } }