Patch up fetching dates from source

server/src/main/java/org/elasticsearch/common/time/DateUtils.java

@@ -194,7 +194,7 @@ public static ZoneId of(String zoneId) {
         return ZoneId.of(zoneId).normalized();
     }
 
-    static final Instant MAX_NANOSECOND_INSTANT = Instant.parse("2262-04-11T23:47:16.854775807Z");
+    public static final Instant MAX_NANOSECOND_INSTANT = Instant.parse("2262-04-11T23:47:16.854775807Z");
 
     static final long MAX_NANOSECOND_IN_MILLIS = MAX_NANOSECOND_INSTANT.toEpochMilli();
 

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -39,6 +39,10 @@
 import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
+import java.math.BigDecimal;
+import java.text.DecimalFormat;
+import java.text.DecimalFormatSymbols;
+import java.text.NumberFormat;
 import java.time.DateTimeException;
 import java.time.Instant;
 import java.time.ZoneId;
@@ -345,6 +349,19 @@ public long parse(String value) {
             return resolution.convert(DateFormatters.from(dateTimeFormatter().parse(value), dateTimeFormatter().locale()).toInstant());
         }
 
+        /**
+         * Format to use to resolve {@link Number}s from the source. Its valid
+         * to send the numbers with up to six digits after the decimal place
+         * and we'll parse them as {@code millis.nanos}. The source
+         * deseralization code isn't particularly careful here and can return
+         * {@link double} instead of the exact string in the {@code _source}.
+         * So we have to *get* that string.
+         */
+        private static final NumberFormat NUMBER_FORMAT = NumberFormat.getInstance(Locale.ROOT);
+        static {
+            NUMBER_FORMAT.setGroupingUsed(false);
+            NUMBER_FORMAT.setMaximumFractionDigits(6);
+        }
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             DateFormatter defaultFormatter = dateTimeFormatter();
@@ -355,7 +372,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
             return new SourceValueFetcher(name(), context, nullValue) {
                 @Override
                 public String parseSourceValue(Object value) {
-                    String date = value.toString();
+                    String date = value instanceof Number ? NUMBER_FORMAT.format(value) : value.toString();
                     long timestamp = parse(date);
                     ZonedDateTime dateTime = resolution().toInstant(timestamp).atZone(ZoneOffset.UTC);
                     return formatter.format(dateTime);

server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.index.mapper;
 
+import com.carrotsearch.randomizedtesting.annotations.Seed;
+
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.Version;
@@ -19,6 +21,7 @@
 import org.elasticsearch.search.DocValueFormat;
 
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
@@ -32,6 +35,7 @@
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.notNullValue;
 
+@Seed("94E9FBDD2E636DE7:1821691D7FEC43C7")
 public class DateFieldMapperTests extends MapperTestCase {
 
     @Override
@@ -356,4 +360,49 @@ public void testResolutionRounding() {
             assertThat(up, equalTo(0L));
         }
     }
+
+    public void testExtractMillis() throws IOException {
+        MapperService mapperService = createMapperService(mapping(b -> b.startObject("field").field("type", "date").endObject()));
+        assertFetch(mapperService, "field", randomLongBetween(0, Long.MAX_VALUE), null);
+        assertFetch(mapperService, "field", randomLongBetween(0, Long.MAX_VALUE), "epoch_millis");
+        assertFetch(mapperService, "field", randomLongBetween(0, Long.MAX_VALUE), "iso8601");
+
+        // The max date iso8601 can parse. It'll format much larger dates.
+        long maxIsoDate = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("9999-12-12T23:59:59.999Z");
+        assertFetch(
+            mapperService,
+            "field",
+            DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(randomLongBetween(0, maxIsoDate)),
+            "iso8601"
+        );
+
+        assertFetch(mapperService, "field", randomIs8601Nanos(maxIsoDate), null);
+        assertFetch(mapperService, "field", randomIs8601Nanos(maxIsoDate), "strict_date_optional_time_nanos");
+        assertFetch(mapperService, "field", randomDecimalNanos(maxIsoDate), null);
+        assertFetch(mapperService, "field", new BigDecimal(randomDecimalNanos(maxIsoDate)), null);
+    }
+
+    public void testExtractNanos() throws IOException {
+        MapperService mapperService = createMapperService(mapping(b -> b.startObject("field").field("type", "date_nanos").endObject()));
+        long maxMillis = DateUtils.MAX_NANOSECOND_INSTANT.toEpochMilli();
+        assertFetch(mapperService, "field", randomLongBetween(0, maxMillis), null);
+        assertFetch(mapperService, "field", randomLongBetween(0, maxMillis), "epoch_millis");
+        assertFetch(mapperService, "field", randomLongBetween(0, maxMillis), "iso8601");
+        assertFetch(mapperService, "field", randomIs8601Nanos(maxMillis), null);
+        assertFetch(mapperService, "field", randomIs8601Nanos(maxMillis), "strict_date_optional_time_nanos");
+        assertFetch(mapperService, "field", randomDecimalNanos(maxMillis), null);
+        assertFetch(mapperService, "field", new BigDecimal(randomDecimalNanos(maxMillis)), null);
+        long maxMillisDoubleKeepsPrecision = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("1970-04-15T05:59:59.999Z");
+        assertFetch(mapperService, "field", new BigDecimal(randomDecimalNanos(maxMillisDoubleKeepsPrecision)), "strict_date_optional_time_nanos");
+    }
+
+    private String randomIs8601Nanos(long maxMillis) {
+        String date = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.formatMillis(randomLongBetween(0, maxMillis));
+        date = date.substring(0, date.length() - 1);  // Strip off trailing "Z"
+        return date + String.format("%06d", between(0, 999999)) + "Z";  // Add nanos and the "Z"
+    }
+
+    private String randomDecimalNanos(long maxMillis) {
+        return Long.toString(randomLongBetween(0, maxMillis)) + "." + between(0, 999999);
+    }
 }

server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java

@@ -458,4 +458,10 @@ public void testSplitQueriesOnWhitespace() throws IOException {
             new String[] { "hello world" }
         );
     }
+
+    public void testExtract() throws IOException {
+        MapperService mapperService = createMapperService(mapping(b -> b.startObject("field").field("type", "keyword").endObject()));
+        assertFetch(mapperService, "field", randomAlphaOfLength(5), null);
+        assertFetch(mapperService, "field", new String[] {randomAlphaOfLength(5), randomAlphaOfLength(5), randomAlphaOfLength(5)}, null);
+    }
 }

test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java

@@ -33,20 +33,24 @@
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.lookup.SearchLookup;
+import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Base class for testing {@link Mapper}s.
@@ -452,4 +456,26 @@ public final void testTextSearchInfoConsistency() throws IOException {
     protected void assertSearchable(MappedFieldType fieldType) {
         assertEquals(fieldType.isSearchable(), fieldType.getTextSearchInfo() != TextSearchInfo.NONE);
     }
+
+    // NOCOMMIT javadoc
+    protected void assertFetch(MapperService mapperService, String field, Object value, String format) throws IOException {
+        MappedFieldType ft = mapperService.fieldType(field);
+        SourceToParse source = source(b -> b.field(ft.name(), value));
+        ValueFetcher docValueFetcher = new DocValueFetcher(
+            ft.docValueFormat(format, null),
+            ft.fielddataBuilder("test", () -> null).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())
+        );
+        SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class);
+        when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field));
+        ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format);
+        ParsedDocument doc = mapperService.documentMapper().parse(source);
+        withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> {
+            SourceLookup sourceLookup = new SourceLookup();
+            sourceLookup.setSegmentAndDocument(ir.leaves().get(0), 0);
+            docValueFetcher.setNextReader(ir.leaves().get(0));
+            List<?> fromDocValues = docValueFetcher.fetchValues(sourceLookup);
+            List<?> fromNative = nativeFetcher.fetchValues(sourceLookup);
+            assertEquals(fromDocValues, fromNative);
+        });
+    }
 }