From f5ed6e054417353bcd239dfe3159503c6d9fb891 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Mar 2021 15:57:30 -0400 Subject: [PATCH] Super randomized tests for fetch fields API (backport of #70278) (#70834) We've had a few bugs in the fields API where is doesn't behave like we'd expect. Typically this happens because it isn't obvious what we expct. So we'll try and use randomized testing to ferret out what we want. This adds a test for most field types that asserts that `fields` works similarly to `docvalues_fields`. We expect this to be true for most fields. It does so by forcing all subclasses of `MapperTestCase` to define a method that makes random values. It declares a few other hooks that subclasses can override to further randomize the test. We skip the test for a few field types that don't have doc values: * `annotated_text` * `completion` * `search_as_you_type` * `text` We should come up with some way to test these without doc values, even if it isn't as nice. But that is a problem for another time, I think. We skip the test for a few more types just because I wanted to cut this PR in half so we could get to reviewing it earlier. We'll get to those in a follow up change. I've filed a few bugs for things that are inconsistent with `docvalues_fields`. Typically that means that we have to limit the random values that we generate to those that *do* round trip properly. --- .../mapper/RankFeatureFieldMapperTests.java | 6 + .../mapper/RankFeaturesFieldMapperTests.java | 6 + .../mapper/ScaledFloatFieldMapperTests.java | 27 ++++ .../SearchAsYouTypeFieldMapperTests.java | 6 + .../mapper/TokenCountFieldMapperTests.java | 16 +++ .../ICUCollationKeywordFieldMapper.java | 2 +- .../ICUCollationKeywordFieldMapperTests.java | 10 ++ .../AnnotatedTextFieldMapperTests.java | 6 + .../murmur3/Murmur3FieldMapperTests.java | 6 + .../elasticsearch/index/mapper/RangeType.java | 10 +- .../index/mapper/BinaryFieldMapperTests.java | 12 ++ .../index/mapper/BooleanFieldMapperTests.java | 16 +++ .../index/mapper/ByteFieldMapperTests.java | 11 ++ .../mapper/CompletionFieldMapperTests.java | 5 + .../index/mapper/DateFieldMapperTests.java | 45 +++++++ .../index/mapper/DoubleFieldMapperTests.java | 11 ++ .../index/mapper/FloatFieldMapperTests.java | 11 ++ .../mapper/GeoPointFieldMapperTests.java | 6 + .../mapper/GeoShapeFieldMapperTests.java | 6 + .../mapper/HalfFloatFieldMapperTests.java | 13 ++ .../index/mapper/IntegerFieldMapperTests.java | 11 ++ .../index/mapper/IpFieldMapperTests.java | 6 + .../index/mapper/KeywordFieldMapperTests.java | 37 +++--- .../LegacyGeoShapeFieldMapperTests.java | 6 + .../index/mapper/LongFieldMapperTests.java | 17 +++ .../index/mapper/NumberFieldMapperTests.java | 7 + .../index/mapper/RangeFieldMapperTests.java | 8 ++ .../index/mapper/ShortFieldMapperTests.java | 11 ++ .../index/mapper/TextFieldMapperTests.java | 11 ++ .../flattened/FlattenedFieldMapperTests.java | 7 + .../AbstractNumericFieldMapperTestCase.java | 7 + .../index/mapper/MapperTestCase.java | 120 ++++++++++++++++-- .../mapper/HistogramFieldMapperTests.java | 7 + ...AggregateDoubleMetricFieldMapperTests.java | 6 + .../ConstantKeywordFieldMapperTests.java | 10 ++ .../UnsignedLongFieldMapperTests.java | 23 ++++ .../VersionStringFieldMapperTests.java | 22 ++++ ...GeoShapeWithDocValuesFieldMapperTests.java | 6 + .../index/mapper/PointFieldMapperTests.java | 6 + .../index/mapper/ShapeFieldMapperTests.java | 7 + .../mapper/DenseVectorFieldMapperTests.java | 7 + .../mapper/SparseVectorFieldMapperTests.java | 7 + .../mapper/WildcardFieldMapperTests.java | 5 + 43 files changed, 550 insertions(+), 35 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index d08d44aa2e009..b691eac6ffcfc 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -136,4 +136,10 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep assertEquals("[rank_feature] fields do not support indexing multiple values for the same field [foo.field] in the same document", e.getCause().getMessage()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java index 0300411bb0298..8a96bd1d4f5e6 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java @@ -134,4 +134,10 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep assertEquals("[rank_features] fields do not support indexing multiple values for the same rank feature [foo.field.bar] in " + "the same document", e.getCause().getMessage()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index c6b29e7b24b5c..18d1b27b1f84d 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -271,4 +271,31 @@ public void testRejectIndexOptions() { containsString("Failed to parse mapping [_doc]: Field [scaling_factor] is required")); assertWarnings("Parameter [index_options] has no effect on type [scaled_float] and will be removed in future"); } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + // Large floats are a terrible idea but the round trip should still work no matter how badly you configure the field + b.field("type", "scaled_float").field("scaling_factor", randomDoubleBetween(0, Float.MAX_VALUE, true)); + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + /* + * randomDoubleBetween will smear the random values out across a huge + * range of valid values. + */ + double v = randomDoubleBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true); + switch (between(0, 3)) { + case 0: + return v; + case 1: + return (float) v; + case 2: + return Double.toString(v); + case 3: + return Float.toString((float) v); + default: + throw new IllegalArgumentException(); + } + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 9d8c9216c62dc..8a024d6b8fee2 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -750,4 +750,10 @@ private static PrefixFieldMapper getPrefixFieldMapper(DocumentMapper defaultMapp assertThat(mapper, instanceOf(PrefixFieldMapper.class)); return (PrefixFieldMapper) mapper; } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We don't have doc values or fielddata", true); + return null; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java index 00baaa5ff5392..5e399f8eaddab 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java @@ -183,4 +183,20 @@ private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse return mapper.parse(request) .docs().stream().findFirst().orElseThrow(() -> new IllegalStateException("Test object not parsed")); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + int words = between(1, 1000); + StringBuilder b = new StringBuilder(words * 5); + b.append(randomAlphaOfLength(4)); + for (int w = 1; w < words; w++) { + b.append(' ').append(randomAlphaOfLength(4)); + } + return b.toString(); + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", "token_count").field("analyzer", "standard"); + } } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 907f9b2158273..608b374e4d76d 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -137,7 +137,7 @@ public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxD throw new UnsupportedOperationException("[regexp] queries are not supported on [" + CONTENT_TYPE + "] fields."); } - public static DocValueFormat COLLATE_FORMAT = new DocValueFormat() { + public static final DocValueFormat COLLATE_FORMAT = new DocValueFormat() { @Override public String getWriteableName() { return "collate"; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index f9bc27f83b727..ba6ba055fb5bc 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -10,6 +10,7 @@ import com.ibm.icu.text.Collator; import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.util.ULocale; + import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -274,4 +275,13 @@ public void testUpdateIgnoreAbove() throws IOException { assertEquals(0, fields.length); } + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + assumeFalse("docvalue_fields is broken", true); + // https://github.com/elastic/elasticsearch/issues/70276 + /* + * docvalue_fields loads garbage bytes. + */ + return null; + } } diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 862bf46397db8..b2bf055c6dcc7 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.analysis.StandardTokenizerFactory; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; @@ -559,4 +560,9 @@ public void testAnalyzedFieldPositionIncrementWithoutPositions() { } } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("annotated_text doesn't have fielddata so we can't check against anything here.", true); + return null; + } } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index d4755d587affc..18229e249c8c3 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.IndexableField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.plugin.mapper.MapperMurmur3Plugin; @@ -55,4 +56,9 @@ public void testDefaults() throws Exception { assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType()); } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java index 957e1094d6be1..0c979fde1ee5d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java @@ -546,6 +546,10 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu } }; + public final String name; + private final NumberFieldMapper.NumberType numberType; + public final LengthType lengthType; + RangeType(String name, LengthType lengthType) { this.name = name; this.numberType = null; @@ -700,9 +704,9 @@ public final Mapper.TypeParser parser() { new RangeFieldMapper.Builder(n, this, RangeFieldMapper.COERCE_SETTING.get(c.getSettings()), c.indexVersionCreated())); } - public final String name; - private final NumberFieldMapper.NumberType numberType; - public final LengthType lengthType; + NumberFieldMapper.NumberType numberType() { + return numberType; + } public enum LengthType { FIXED_4 { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index b5cfdebc67c6e..0cbe10cff3c55 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -118,4 +118,16 @@ public void testStoredValue() throws IOException { assertEquals(new BytesArray(value), originalValue); } } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We can't parse the binary doc values we send", true); + // AwaitsFix https://github.com/elastic/elasticsearch/issues/70244 + return null; + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", "binary").field("doc_values", true); // enable doc_values so the test is happy + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index dbee9bead4e40..0aad098b7e19e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -181,4 +181,20 @@ public void testBoosts() throws Exception { assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null)); assertParseMaximalWarnings(); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + switch (between(0, 3)) { + case 0: + return randomBoolean(); + case 1: + return randomBoolean() ? "true" : "false"; + case 2: + return randomBoolean() ? "true" : ""; + case 3: + return randomBoolean() ? "true" : null; + default: + throw new IllegalStateException(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java index aa7952531cb8b..bca1e35b1c893 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java @@ -36,4 +36,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "byte"); } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomByte(); + } + if (randomBoolean()) { + return randomDouble(); + } + return randomDoubleBetween(Byte.MIN_VALUE, Byte.MAX_VALUE, true); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 119f4ff8b13ad..e24ba0e526128 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -764,4 +764,9 @@ protected V featureValueOf(T actual) { }; } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We don't have doc values or fielddata", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 2dfcf1b5020a2..30ec247bc35e3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.search.DocValueFormat; @@ -496,6 +497,50 @@ public void testFetchNanosFromFixedNanosFormatted() throws IOException { ); } + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", randomBoolean() ? "date" : "date_nanos"); + } + + @Override + protected String randomFetchTestFormat() { + // TODO more choices! The test should work fine even for choices that throw out a ton of precision. + switch (randomInt(2)) { + case 0: + return null; + case 1: + return "epoch_millis"; + case 2: + return "iso8601"; + default: + throw new IllegalStateException(); + } + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + switch (((DateFieldType) ft).resolution()) { + case MILLISECONDS: + if (randomBoolean()) { + return randomIs8601Nanos(MAX_ISO_DATE); + } + return randomLongBetween(0, Long.MAX_VALUE); + case NANOSECONDS: + switch (randomInt(2)) { + case 0: + return randomLongBetween(0, MAX_NANOS); + case 1: + return randomIs8601Nanos(MAX_NANOS); + case 2: + return new BigDecimal(randomDecimalNanos(MAX_MILLIS_DOUBLE_NANOS_KEEPS_PRECISION)); + default: + throw new IllegalStateException(); + } + default: + throw new IllegalStateException(); + } + } + private MapperService dateNanosMapperService() throws IOException { return createMapperService(mapping(b -> b.startObject("field").field("type", "date_nanos").endObject())); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java index 46a73538c7fec..0fade027daf0a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java @@ -37,4 +37,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "double"); } + + @Override + protected Number randomNumber() { + /* + * The source parser and doc values round trip will both increase + * the precision to 64 bits if the value is less precise. + * randomDoubleBetween will smear the values out across a wide + * range of valid values. + */ + return randomBoolean() ? randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true) : randomFloat(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java index f448f402263e2..99eba05cc9530 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java @@ -37,4 +37,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "float"); } + + @Override + protected Number randomNumber() { + /* + * The source parser and doc values round trip will both reduce + * the precision to 32 bits if the value is more precise. + * randomDoubleBetween will smear the values out across a wide + * range of valid values. + */ + return randomBoolean() ? randomDoubleBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true) : randomFloat(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 67016c5f6e80d..1933f958f5e54 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -340,4 +340,10 @@ protected void assertSearchable(MappedFieldType fieldType) { //always searchable even if it uses TextSearchInfo.NONE assertTrue(fieldType.isSearchable()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 4c27a6011c268..d8ceb4870681b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -226,4 +226,10 @@ protected void assertSearchable(MappedFieldType fieldType) { //always searchable even if it uses TextSearchInfo.NONE assertTrue(fieldType.isSearchable()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java index 4e9692e990242..4ad27801a1d76 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.HalfFloatPoint; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; @@ -39,4 +40,16 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "half_float"); } + + @Override + protected Number randomNumber() { + /* + * The native valueFetcher returns 32 bits of precision but the + * doc values fetcher returns 16 bits of precision. To make it + * all line up we round here instead of in the fetcher. This bug + * is tracked in: + * https://github.com/elastic/elasticsearch/issues/70260 + */ + return HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java index 61d3de7bfcce8..9d088466e7891 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java @@ -37,4 +37,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "integer"); } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomInt(); + } + if (randomBoolean()) { + return randomDouble(); + } + return randomDoubleBetween(Integer.MIN_VALUE, Integer.MAX_VALUE, true); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 4411936cf73ba..abbeea181449d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.termvectors.TermVectorsService; @@ -193,4 +194,9 @@ public void testNullValue() throws IOException { })); assertWarnings("Error parsing [:1] as IP in [null_value] on field [field]); [null_value] will be ignored"); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + return NetworkAddress.format(randomIp(randomBoolean())); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 1b75ae24e08d9..99b46ed545a72 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -40,14 +40,11 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static java.util.stream.Collectors.toList; import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -474,25 +471,21 @@ public void testSplitQueriesOnWhitespace() throws IOException { ); } - public void testFetch() throws IOException { - assertFetch(keywordMapperService(), "field", randomAlphaOfLength(5), null); - } - - public void testFetchMany() throws IOException { - /* - * When we have many values doc values will sort and unique them. - * Source fetching won't, but we expect that. So we test with - * sorted and uniqued values. - */ - int count = between(2, 10); - Set values = new HashSet<>(); - while (values.size() < count) { - values.add(randomAlphaOfLength(5)); + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + switch (between(0, 3)) { + case 0: + return randomAlphaOfLengthBetween(1, 100); + case 1: + return randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100); + case 2: + return randomLong(); + case 3: + return randomDouble(); + case 4: + return randomBoolean(); + default: + throw new IllegalStateException(); } - assertFetch(keywordMapperService(), "field", values.stream().sorted().collect(toList()), null); - } - - private MapperService keywordMapperService() throws IOException { - return createMapperService(mapping(b -> b.startObject("field").field("type", "keyword").endObject())); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java index c456d1f571e5b..f3002e259f334 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -634,4 +634,10 @@ protected void assertSearchable(MappedFieldType fieldType) { //always searchable even if it uses TextSearchInfo.NONE assertTrue(fieldType.isSearchable()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java index 3bb6df31629a0..e323a50e293df 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java @@ -60,4 +60,21 @@ public void testLongIndexingCoercesIntoRange() throws Exception { doc = mapper.parse(source(b -> b.field("field", "-9223372036854775808.9"))); assertThat(doc.rootDoc().getFields("field"), arrayWithSize(2)); } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomLong(); + } + if (randomBoolean()) { + return randomDouble(); + } + assumeFalse("https://github.com/elastic/elasticsearch/issues/70585", true); + return randomDoubleBetween(Long.MIN_VALUE, Long.MAX_VALUE, true); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/70585") + public void testFetchCoerced() throws IOException { + assertFetch(randomFetchTestMapper(), "field", 3.783147882954537E18, randomFetchTestFormat()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 9238b6aa63226..6bce801ce7484 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -218,6 +218,13 @@ public void testOutOfRangeValues() throws IOException { e.getCause().getMessage(), containsString(item.message)); } } + } + @Override + protected final Object generateRandomInputValue(MappedFieldType ft) { + Number n = randomNumber(); + return randomBoolean() ? n : n.toString(); } + + protected abstract Number randomNumber(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index c63219886234b..547aec31ff58b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -359,4 +359,12 @@ public void testIllegalFormatField() throws Exception { assertThat(e.getMessage(), containsString("Invalid format: [[test_format]]: Unknown pattern letter: t")); } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + // Doc value fetching crashes. + // https://github.com/elastic/elasticsearch/issues/70269 + // TODO when we fix doc values fetcher we should add tests for date and ip ranges. + assumeFalse("DocValuesFetcher doesn't work", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java index 411fb91532cfa..c01ac89b82bdd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java @@ -37,4 +37,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "short"); } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomShort(); + } + if (randomBoolean()) { + return randomDouble(); + } + return randomDoubleBetween(Short.MIN_VALUE, Short.MAX_VALUE, true); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index f18e854b67402..d33c202a7d0dc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1091,4 +1091,15 @@ public void testSimpleMerge() throws IOException { assertThat(mapperService.documentMapper().mappers().getMapper("field"), instanceOf(TextFieldMapper.class)); assertThat(mapperService.documentMapper().mappers().getMapper("other_field"), instanceOf(KeywordFieldMapper.class)); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We don't have a way to assert things here", true); + return null; + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + assumeFalse("We don't have a way to assert things here", true); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index f70d213078d11..4f920837ac431 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; @@ -333,4 +334,10 @@ public void testSplitQueriesOnWhitespace() throws IOException { assertTokenStreamContents(keyedFieldType.getTextSearchInfo().getSearchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"Hello", "World"}); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java index 08a4e322e3174..dd6eb95a9d7ce 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java @@ -7,6 +7,8 @@ */ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.xcontent.XContentBuilder; + import java.io.IOException; import java.util.Set; @@ -77,4 +79,9 @@ public final void testNullValue() throws IOException { } protected abstract void doTestNullValue(String type) throws IOException; + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", randomFrom(types())); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 87f09702cee30..de219b07c643f 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -44,7 +44,9 @@ import java.util.function.Consumer; import java.util.function.Supplier; +import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.mock; @@ -286,16 +288,11 @@ public final void testDeprecatedBoost() throws IOException { protected final List fetchFromDocValues(MapperService mapperService, MappedFieldType ft, DocValueFormat format, Object sourceValue) throws IOException { - BiFunction, IndexFieldData> fieldDataLookup = (mft, lookupSource) -> mft - .fielddataBuilder("test", () -> { - throw new UnsupportedOperationException(); - }) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); SetOnce> result = new SetOnce<>(); withLuceneIndex(mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft)); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); @@ -445,6 +442,82 @@ protected void assertSearchable(MappedFieldType fieldType) { assertEquals(fieldType.isSearchable(), fieldType.getTextSearchInfo() != TextSearchInfo.NONE); } + /** + * Asserts that fetching a single value from doc values and from the native + * {@link MappedFieldType#valueFetcher} produce the same results. + *

+ * Generally this method covers many many random cases but rarely. So if + * it fails its generally a good idea to capture its randomized + * parameters into a new method so we can be sure we consistently test + * any unique and interesting failure case. See the tests for + * {@link DateFieldMapper} for some examples. + */ + public final void testFetch() throws IOException { + MapperService mapperService = randomFetchTestMapper(); + try { + MappedFieldType ft = mapperService.fieldType("field"); + assertFetch(mapperService, "field", generateRandomInputValue(ft), randomFetchTestFormat()); + } finally { + assertParseMinimalWarnings(); + } + } + + /** + * Asserts that fetching many values from doc values and from the native + * {@link MappedFieldType#valueFetcher} produce the same results. + *

+ * Generally this method covers many many random cases but rarely. So if + * it fails its generally a good idea to capture its randomized + * parameters into a new method so we can be sure we consistently test + * any unique and interesting failure case. See the tests for + * {@link DateFieldMapper} for some examples. + */ + public final void testFetchMany() throws IOException { + MapperService mapperService = randomFetchTestMapper(); + try { + MappedFieldType ft = mapperService.fieldType("field"); + int count = between(2, 10); + List values = new ArrayList<>(count); + while (values.size() < count) { + values.add(generateRandomInputValue(ft)); + } + assertFetch(mapperService, "field", values, randomFetchTestFormat()); + } finally { + assertParseMinimalWarnings(); + } + } + + protected final MapperService randomFetchTestMapper() throws IOException { + return createMapperService(mapping(b -> { + b.startObject("field"); + randomFetchTestFieldConfig(b); + b.endObject(); + })); + } + + /** + * Field configuration for {@link #testFetch} and {@link #testFetchMany}. + * Default implementation delegates to {@link #minimalMapping} but can + * be overridden to randomize the field type and options. + */ + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + minimalMapping(b); + } + + /** + * A random format to use when tripping in {@link #testFetch} and + * {@link #testFetchMany}. + */ + protected String randomFetchTestFormat() { + return null; + } + + /** + * Create a random {@code _source} value for this field. Must be compatible + * with {@link XContentBuilder#value(Object)} and the field's parser. + */ + protected abstract Object generateRandomInputValue(MappedFieldType ft); + /** * Assert that fetching a value using {@link MappedFieldType#valueFetcher} * produces the same value as fetching using doc values. @@ -458,15 +531,44 @@ protected void assertFetch(MapperService mapperService, String field, Object val ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.sourcePath(field)).thenReturn(org.elasticsearch.common.collect.Set.of(field)); + when(searchExecutionContext.getForField(ft)).thenAnswer( + inv -> { return fieldDataLookup().apply(ft, () -> { throw new UnsupportedOperationException(); }); } + ); ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSegmentAndDocument(ir.leaves().get(0), 0); docValueFetcher.setNextReader(ir.leaves().get(0)); - List fromDocValues = docValueFetcher.fetchValues(sourceLookup); - List fromNative = nativeFetcher.fetchValues(sourceLookup); - assertEquals(fromDocValues, fromNative); + nativeFetcher.setNextReader(ir.leaves().get(0)); + List fromDocValues = docValueFetcher.fetchValues(sourceLookup); + List fromNative = nativeFetcher.fetchValues(sourceLookup); + /* + * The native fetcher uses byte, short, etc but doc values always + * uses long or double. This difference is fine because on the outside + * users can't see it. + */ + fromNative = fromNative.stream().map(o -> { + if (o instanceof Integer || o instanceof Short || o instanceof Byte) { + return ((Number) o).longValue(); + } + if (o instanceof Float) { + return ((Float) o).doubleValue(); + } + return o; + }).collect(toList()); + /* + * Doc values sort according to something appropriate to the field + * and the native fetchers usually don't sort. We're ok with this + * difference. But we have to convince the test we're ok with it. + */ + assertThat("fetching " + value, fromNative, containsInAnyOrder(fromDocValues.toArray())); }); } + + private BiFunction, IndexFieldData> fieldDataLookup() { + return (mft, lookupSource) -> mft + .fielddataBuilder("test", lookupSource) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 3b9a3a1e2832b..1272fd3d804a2 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; @@ -293,4 +294,10 @@ public void testNegativeCount() throws Exception { Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source)); assertThat(e.getCause().getMessage(), containsString("[counts] elements must be >= 0 but got -3")); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java index ab3a13ad36366..184c3b2dc5c72 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -528,4 +528,10 @@ protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseCo assertDocValuesField(fields, "field." + defaultMetric); assertNoFieldNamesField(fields); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index a755f590ef838..d3ac24c2f281a 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -162,4 +162,14 @@ protected void registerParameters(ParameterChecker checker) throws IOException { b.field("value", "bar"); })); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + return ((ConstantKeywordFieldType) ft).value(); + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", "constant_keyword").field("value", randomAlphaOfLengthBetween(1, 10)); + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index a2bb8352dc105..cfe0ad0c18feb 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; @@ -327,4 +328,26 @@ public void testExistsQueryDocValuesDisabled() throws IOException { assertParseMinimalWarnings(); } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + Number n = randomNumericValue(); + return randomBoolean() ? n : n.toString(); + } + + private Number randomNumericValue() { + switch (randomInt(8)) { + case 0: + return randomNonNegativeByte(); + case 1: + return (short) between(0, Short.MAX_VALUE); + case 2: + return randomInt(Integer.MAX_VALUE); + case 3: + case 4: + return randomNonNegativeLong(); + default: + BigInteger big = BigInteger.valueOf(randomLongBetween(0, Long.MAX_VALUE)).shiftLeft(1); + return big.add(randomBoolean() ? BigInteger.ONE : BigInteger.ZERO); + } + } } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 3941a0567aef1..a109298ba4bd3 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; @@ -129,4 +130,25 @@ public void testFailsParsingNestedList() throws IOException { ex.getMessage() ); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + return randomVersionNumber() + (randomBoolean() ? "" : randomPrerelease()); + } + + private String randomVersionNumber() { + int numbers = between(1, 3); + String v = Integer.toString(between(0, 100)); + for (int i = 1; i < numbers; i++) { + v += "." + between(0, 100); + } + return v; + } + + private String randomPrerelease() { + if (rarely()) { + return randomFrom("alpha", "beta", "prerelease", "whatever"); + } + return randomFrom("alpha", "beta", "") + randomVersionNumber(); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index 86dda9de50aef..a8c82ca4a0d50 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -323,4 +323,10 @@ public String toXContentString(GeoShapeWithDocValuesFieldMapper mapper) throws I protected void assertSearchable(MappedFieldType fieldType) { } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index d49d0cd6030f7..5b51f2ad9a4a3 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; @@ -211,4 +212,9 @@ public void testIgnoreZValue() throws IOException { } } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 240835263ac82..ed91cc2652e4b 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; @@ -235,4 +236,10 @@ public String toXContentString(ShapeFieldMapper mapper, boolean includeDefaults) public String toXContentString(ShapeFieldMapper mapper) { return toXContentString(mapper, true); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java index 94bbded60cda0..55dc4a915144c 100644 --- a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.common.collect.List; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; @@ -154,4 +155,10 @@ public void testDocumentsWithIncorrectDims() throws Exception { () -> mapper.parse(source(b -> b.array("field", invalidVector2)))); assertThat(e2.getCause().getMessage(), containsString("has number of dimensions [2] less than defined in the mapping [3]")); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java index f8e6227e0ef5a..33f22c84d393b 100644 --- a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapperTests.java @@ -14,6 +14,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; @@ -234,4 +235,10 @@ public void testMeta() throws IOException { super.testMeta(); assertParseMinimalWarnings(); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("doesn't support docvalues_fetcher", true); + return null; + } } diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 861b7c8368546..2400c3baaf5c0 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -984,4 +984,9 @@ private String getRandomWildcardPattern() { } return sb.toString(); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + return randomAlphaOfLengthBetween(1, 100); + } }