Use mappings to format doc-value fields by default.

docs/reference/migration/migrate_7_0/search.asciidoc

@@ -85,6 +85,15 @@ for a particular index with the index setting `index.max_regex_length`.
 Search requests with extra content after the main object will no longer be accepted
 by the `_search` endpoint. A parsing exception will be thrown instead.
 
+==== Doc-value fields default format
+
+The format of doc-value fields is changing to be the same as what could be
+obtained in 6.x with the special `use_field_mapping` format. This is mostly a
+change for date fields, which are now formatted based on the format that is
+configured in the mappings by default. This behavior can be changed by
+specifying a <<search-request-docvalue-fields,`format`>> with the doc-value
+field.
+
 ==== Semantics changed for `max_concurrent_shard_requests`
 
 `max_concurrent_shard_requests` used to limit the total number of concurrent shard

docs/reference/search/request/docvalue-fields.asciidoc

@@ -12,9 +12,9 @@ GET /_search
         "match_all": {}
     },
     "docvalue_fields" : [
+        "my_ip_field", <1>
         {
-            "field": "my_ip_field", <1>
-            "format": "use_field_mapping" <2>
+            "field": "my_keyword_field" <2>
         },
         {
             "field": "my_date_field",
@@ -25,13 +25,10 @@ GET /_search
 --------------------------------------------------
 // CONSOLE
 <1> the name of the field
-<2> the special `use_field_mapping` format tells Elasticsearch to use the format from the mapping
-<3> date fields may use a custom format
+<2> an object notation is supported as well
+<3> the object notation allows to specify a custom format
 
-Doc value fields can work on fields that are not stored.
-
-Note that if the fields parameter specifies fields without docvalues it will try to load the value from the fielddata cache
-causing the terms for that field to be loaded to memory (cached), which will result in more memory consumption.
+Doc value fields can work on fields that have doc-values enabled, regardless of whether they are stored.
 
 [float]
 ==== Custom formats
@@ -40,9 +37,8 @@ While most fields do not support custom formats, some of them do:
  - <<date,Date>> fields can take any <<mapping-date-format,date format>>.
  - <<number,Numeric>> fields accept a https://docs.oracle.com/javase/8/docs/api/java/text/DecimalFormat.html[DecimalFormat pattern].
 
-All fields support the special `use_field_mapping` format, which tells
-Elasticsearch to use the mappings to figure out a default format.
+By default fields are formatted based on a sensible configuration that depends
+on their mappings: `long`, `double` and other numeric fields are formatted as
+numbers, `keyword` fields are formatted as strings, `date` fields are formatted
+with the configured `date` format, etc.
 
-NOTE: The default is currently to return the same output as
-<<search-request-script-fields,script fields>>. However it will change in 7.0
-to behave as if the `use_field_mapping` format was provided.

docs/reference/search/request/inner-hits.asciidoc

@@ -243,10 +243,7 @@ POST test/_search
       "inner_hits": {
         "_source" : false,
         "docvalue_fields" : [
-          {
-            "field": "comments.text.keyword",
-            "format": "use_field_mapping"
-          }
+          "comments.text.keyword"
         ]
       }
     }

rest-api-spec/src/main/resources/rest-api-spec/test/search.inner_hits/10_basic.yml

@@ -45,8 +45,8 @@ setup:
 "Nested doc version and seqIDs":
 
     - skip:
-        version: " - 6.3.99"
-        reason:  "object notation for docvalue_fields was introduced in 6.4"
+        version: " - 6.99.99"
+        reason:  "Triggers warnings before 7.0"
 
     - do:
         index:
@@ -60,7 +60,7 @@ setup:
 
     - do:
         search:
-          body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ { "field": "_seq_no", "format": "use_field_mapping" } ]} }}, "version": true, "docvalue_fields" : [ { "field": "_seq_no", "format": "use_field_mapping" } ] }
+          body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ "_seq_no" ] } } }, "version": true, "docvalue_fields" : [ "_seq_no" ] }
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0._index: "test" }
@@ -83,7 +83,7 @@ setup:
 
     - do:
         search:
-          body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ { "field": "_seq_no", "format": "use_field_mapping" } ]} }}, "version": true, "docvalue_fields" : [ { "field": "_seq_no", "format": "use_field_mapping" } ] }
+          body: { "query" : { "nested" : { "path" : "nested_field", "query" : { "match_all" : {} }, "inner_hits" : { version: true, "docvalue_fields": [ "_seq_no" ] } } }, "version": true, "docvalue_fields" : [ "_seq_no" ] }
 
     - match: { hits.total: 1 }
     - match: { hits.hits.0._index: "test" }

rest-api-spec/src/main/resources/rest-api-spec/test/search/10_source_filtering.yml

@@ -134,12 +134,9 @@ setup:
 ---
 "docvalue_fields":
   - skip:
-      version: " - 6.3.99"
-      reason: format option was added in 6.4
-      features: warnings
+      version: " - 6.9.99"
+      reason: Triggers a deprecation warning before 7.0
   - do:
-      warnings:
-        - 'Doc-value field [count] is not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with the doc value field in order to opt in for the future behaviour and ease the migration to 7.0.'
       search:
         body:
           docvalue_fields: [ "count" ]
@@ -148,22 +145,22 @@ setup:
 ---
 "docvalue_fields as url param":
   - skip:
-      version: " - 6.3.99"
-      reason: format option was added in 6.4
-      features: warnings
+      version: " - 6.99.99"
+      reason: Triggered a deprecation warning before 7.0
   - do:
-      warnings:
-        - 'Doc-value field [count] is not using a format. The output will change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass [format=use_field_mapping] with the doc value field in order to opt in for the future behaviour and ease the migration to 7.0.'
       search:
         docvalue_fields: [ "count" ]
   - match: { hits.hits.0.fields.count: [1] }
 
 ---
 "docvalue_fields with default format":
   - skip:
-      version: " - 6.3.99"
-      reason: format option was added in 6.4
+      version: " - 6.99.99"
+      reason: Only triggers warnings on 7.0+
+      features: warnings
   - do:
+      warnings:
+        - "[use_field_mapping] is a special format that was only used to ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore."
       search:
         body:
           docvalue_fields:

rest-api-spec/src/main/resources/rest-api-spec/test/search/30_limits.yml

@@ -63,8 +63,8 @@ setup:
 "Docvalues_fields size limit":
 
   - skip:
-      version: " - 6.3.99"
-      reason: "The object notation for docvalue_fields is only supported on 6.4+"
+      version: " - 6.99.99"
+      reason: "Triggers warnings before 7.0"
   - do:
       catch:      /Trying to retrieve too many docvalue_fields\. Must be less than or equal to[:] \[2\] but was \[3\]\. This limit can be set by changing the \[index.max_docvalue_fields_search\] index level setting\./
       search:
@@ -73,12 +73,9 @@ setup:
           query:
             match_all: {}
           docvalue_fields:
-            - field: "one"
-              format: "use_field_mapping"
-            - field: "two"
-              format: "use_field_mapping"
-            - field: "three"
-              format: "use_field_mapping"
+            - "one"
+            - "two"
+            - "three"
 
 ---
 "Script_fields size limit":

server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsContext.java

@@ -38,8 +38,6 @@
  */
 public class DocValueFieldsContext {
 
-    public static final String USE_DEFAULT_FORMAT = "use_field_mapping";
-
     /**
      * Wrapper around a field name and the format that should be used to
      * display values of this field.

server/src/main/java/org/elasticsearch/search/fetch/subphase/DocValueFieldsFetchSubPhase.java

@@ -28,7 +28,6 @@
 import org.elasticsearch.index.fielddata.AtomicNumericFieldData;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.IndexNumericFieldData;
-import org.elasticsearch.index.fielddata.ScriptDocValues;
 import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
 import org.elasticsearch.index.mapper.MappedFieldType;
@@ -54,6 +53,7 @@
  */
 public final class DocValueFieldsFetchSubPhase implements FetchSubPhase {
 
+    private static final String USE_DEFAULT_FORMAT = "use_field_mapping";
     private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(DocValueFieldsFetchSubPhase.class));
 
     @Override
@@ -64,9 +64,9 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
             String name = context.collapse().getFieldType().name();
             if (context.docValueFieldsContext() == null) {
                 context.docValueFieldsContext(new DocValueFieldsContext(
-                        Collections.singletonList(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT))));
+                        Collections.singletonList(new FieldAndFormat(name, null))));
             } else if (context.docValueFieldsContext().fields().stream().map(ff -> ff.field).anyMatch(name::equals) == false) {
-                context.docValueFieldsContext().fields().add(new FieldAndFormat(name, DocValueFieldsContext.USE_DEFAULT_FORMAT));
+                context.docValueFieldsContext().fields().add(new FieldAndFormat(name, null));
             }
         }
 
@@ -77,28 +77,21 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
         hits = hits.clone(); // don't modify the incoming hits
         Arrays.sort(hits, Comparator.comparingInt(SearchHit::docId));
 
+        boolean usesDefaultFormat = false;
         for (FieldAndFormat fieldAndFormat : context.docValueFieldsContext().fields()) {
             String field = fieldAndFormat.field;
             MappedFieldType fieldType = context.mapperService().fullName(field);
             if (fieldType != null) {
                 final IndexFieldData<?> indexFieldData = context.getForField(fieldType);
-                final DocValueFormat format;
-                if (fieldAndFormat.format == null) {
-                    DEPRECATION_LOGGER.deprecated("Doc-value field [" + fieldAndFormat.field + "] is not using a format. The output will " +
-                            "change in 7.0 when doc value fields get formatted based on mappings by default. It is recommended to pass " +
-                            "[format={}] with the doc value field in order to opt in for the future behaviour and ease the migration to " +
-                            "7.0.", DocValueFieldsContext.USE_DEFAULT_FORMAT);
-                    format = null;
-                } else {
-                    String formatDesc = fieldAndFormat.format;
-                    if (Objects.equals(formatDesc, DocValueFieldsContext.USE_DEFAULT_FORMAT)) {
-                        formatDesc = null;
-                    }
-                    format = fieldType.docValueFormat(formatDesc, null);
+                String formatDesc = fieldAndFormat.format;
+                if (Objects.equals(formatDesc, USE_DEFAULT_FORMAT)) {
+                    // TODO: Remove in 8.x
+                    usesDefaultFormat = true;
+                    formatDesc = null;
                 }
+                final DocValueFormat format = fieldType.docValueFormat(formatDesc, null);
                 LeafReaderContext subReaderContext = null;
                 AtomicFieldData data = null;
-                ScriptDocValues<?> scriptValues = null; // legacy
                 SortedBinaryDocValues binaryValues = null; // binary / string / ip fields
                 SortedNumericDocValues longValues = null; // int / date fields
                 SortedNumericDoubleValues doubleValues = null; // floating-point fields
@@ -108,9 +101,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
                         int readerIndex = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves());
                         subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex);
                         data = indexFieldData.load(subReaderContext);
-                        if (format == null) {
-                            scriptValues = data.getScriptValues();
-                        } else if (indexFieldData instanceof IndexNumericFieldData) {
+                        if (indexFieldData instanceof IndexNumericFieldData) {
                             if (((IndexNumericFieldData) indexFieldData).getNumericType().isFloatingPoint()) {
                                 doubleValues = ((AtomicNumericFieldData) data).getDoubleValues();
                             } else {
@@ -131,10 +122,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
                     final List<Object> values = hitField.getValues();
 
                     int subDocId = hit.docId() - subReaderContext.docBase;
-                    if (scriptValues != null) {
-                        scriptValues.setNextDocId(subDocId);
-                        values.addAll(scriptValues);
-                    } else if (binaryValues != null) {
+                    if (binaryValues != null) {
                         if (binaryValues.advanceExact(subDocId)) {
                             for (int i = 0, count = binaryValues.docValueCount(); i < count; ++i) {
                                 values.add(format.format(binaryValues.nextValue()));
@@ -158,5 +146,9 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
                 }
             }
         }
+        if (usesDefaultFormat) {
+            DEPRECATION_LOGGER.deprecated("[" + USE_DEFAULT_FORMAT + "] is a special format that was only used to " +
+                    "ease the transition to 7.x. It has become the default and shouldn't be set explicitly anymore.");
+        }
     }
 }

server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java

@@ -32,7 +32,6 @@
 import org.elasticsearch.script.ScriptType;
 import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
 import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext.FieldAndFormat;
 import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
 import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilderTests;
@@ -150,8 +149,7 @@ public static InnerHitBuilder randomInnerHits() {
             innerHits.setStoredFieldNames(randomListStuff(16, () -> randomAlphaOfLengthBetween(1, 16)));
         }
         innerHits.setDocValueFields(randomListStuff(16,
-                () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16),
-                        randomBoolean() ? null : DocValueFieldsContext.USE_DEFAULT_FORMAT)));
+                () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null)));
         // Random script fields deduped on their field name.
         Map<String, SearchSourceBuilder.ScriptField> scriptFields = new HashMap<>();
         for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) {
@@ -192,8 +190,7 @@ static InnerHitBuilder mutate(InnerHitBuilder original) throws IOException {
         modifiers.add(() -> {
             if (randomBoolean()) {
                 copy.setDocValueFields(randomValueOtherThan(copy.getDocValueFields(),
-                        () -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16),
-                                randomBoolean() ? null : DocValueFieldsContext.USE_DEFAULT_FORMAT))));
+                        () -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null))));
             } else {
                 copy.addDocValueField(randomAlphaOfLengthBetween(1, 16));
             }

server/src/test/java/org/elasticsearch/search/fields/SearchFieldsIT.java

@@ -19,7 +19,6 @@
 
 package org.elasticsearch.search.fields;
 
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.index.IndexRequestBuilder;
 import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
@@ -802,12 +801,12 @@ public void testDocValueFields() throws Exception {
         assertThat(searchResponse.getHits().getAt(0).getFields().get("long_field").getValue(), equalTo((Object) 4L));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("float_field").getValue(), equalTo((Object) 5.0));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("double_field").getValue(), equalTo((Object) 6.0d));
-        assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(), equalTo(date));
+        assertThat(searchResponse.getHits().getAt(0).getFields().get("date_field").getValue(),
+                equalTo(Joda.forPattern("dateOptionalTime").printer().print(date)));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("boolean_field").getValue(), equalTo((Object) true));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("text_field").getValue(), equalTo("foo"));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
-        assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(),
-                equalTo(new BytesRef(new byte[] {42, 100})));
+        assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
         assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
 
         builder = client().prepareSearch().setQuery(matchAllQuery())

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java

@@ -120,7 +120,7 @@ private SearchRequestBuilder buildSearchRequest(long start) {
             if (docValueField.equals(context.extractedFields.timeField())) {
                 searchRequestBuilder.addDocValueField(docValueField, EPOCH_MILLIS_FORMAT);
             } else {
-                searchRequestBuilder.addDocValueField(docValueField, DocValueFieldsContext.USE_DEFAULT_FORMAT);
+                searchRequestBuilder.addDocValueField(docValueField);
             }
         }
         String[] sourceFields = context.extractedFields.getSourceFields();

x-pack/plugin/sql/sql-proto/src/test/java/org/elasticsearch/xpack/sql/plugin/SqlTranslateResponseTests.java

@@ -6,7 +6,6 @@
 package org.elasticsearch.xpack.sql.plugin;
 
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.elasticsearch.search.fetch.subphase.DocValueFieldsContext;
 import org.elasticsearch.test.AbstractStreamableTestCase;
 
 import java.io.IOException;
@@ -19,7 +18,7 @@ protected SqlTranslateResponse createTestInstance() {
         if (randomBoolean()) {
             long docValues = iterations(5, 10);
             for (int i = 0; i < docValues; i++) {
-                s.docValueField(randomAlphaOfLength(10), DocValueFieldsContext.USE_DEFAULT_FORMAT);
+                s.docValueField(randomAlphaOfLength(10));
             }
         }
 

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/SqlSourceBuilder.java

@@ -69,8 +69,7 @@ public void build(SearchSourceBuilder sourceBuilder) {
         if (!sourceFields.isEmpty()) {
             sourceBuilder.fetchSource(sourceFields.toArray(Strings.EMPTY_ARRAY), null);
         }
-        docFields.forEach(field -> sourceBuilder.docValueField(field.field,
-                field.format == null ? DocValueFieldsContext.USE_DEFAULT_FORMAT : field.format));
+        docFields.forEach(field -> sourceBuilder.docValueField(field.field, field.format));
         scriptFields.forEach(sourceBuilder::scriptField);
     }
 }

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/action/SqlLicenseIT.java

@@ -158,7 +158,7 @@ public void testSqlTranslateActionLicense() throws Exception {
             .query("SELECT * FROM test").get();
         SearchSourceBuilder source = response.source();
         assertThat(source.docValueFields(), Matchers.contains(
-                new DocValueFieldsContext.FieldAndFormat("count", DocValueFieldsContext.USE_DEFAULT_FORMAT)));
+                new DocValueFieldsContext.FieldAndFormat("count", null)));
         FetchSourceContext fetchSource = source.fetchSource();
         assertThat(fetchSource.includes(), Matchers.arrayContaining("data"));
     }