From 91c700bd99e59d099cee282fe5b4a1a266f70748 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 24 Mar 2021 14:16:27 -0400 Subject: [PATCH] Super randomized tests for fetch fields API (#70278) We've had a few bugs in the fields API where is doesn't behave like we'd expect. Typically this happens because it isn't obvious what we expct. So we'll try and use randomized testing to ferret out what we want. This adds a test for most field types that asserts that `fields` works similarly to `docvalues_fields`. We expect this to be true for most fields. It does so by forcing all subclasses of `MapperTestCase` to define a method that makes random values. It declares a few other hooks that subclasses can override to further randomize the test. We skip the test for a few field types that don't have doc values: * `annotated_text` * `completion` * `search_as_you_type` * `text` We should come up with some way to test these without doc values, even if it isn't as nice. But that is a problem for another time, I think. We skip the test for a few more types just because I wanted to cut this PR in half so we could get to reviewing it earlier. We'll get to those in a follow up change. I've filed a few bugs for things that are inconsistent with `docvalues_fields`. Typically that means that we have to limit the random values that we generate to those that *do* round trip properly. --- .../mapper/RankFeatureFieldMapperTests.java | 6 + .../mapper/RankFeaturesFieldMapperTests.java | 6 + .../mapper/ScaledFloatFieldMapperTests.java | 26 ++++ .../SearchAsYouTypeFieldMapperTests.java | 6 + .../mapper/TokenCountFieldMapperTests.java | 16 +++ .../ICUCollationKeywordFieldMapper.java | 2 +- .../ICUCollationKeywordFieldMapperTests.java | 10 ++ .../AnnotatedTextFieldMapperTests.java | 6 + .../murmur3/Murmur3FieldMapperTests.java | 6 + .../elasticsearch/index/mapper/RangeType.java | 10 +- .../index/mapper/BinaryFieldMapperTests.java | 12 ++ .../index/mapper/BooleanFieldMapperTests.java | 16 +++ .../index/mapper/ByteFieldMapperTests.java | 11 ++ .../mapper/CompletionFieldMapperTests.java | 5 + .../index/mapper/DateFieldMapperTests.java | 45 +++++++ .../index/mapper/DoubleFieldMapperTests.java | 11 ++ .../index/mapper/FloatFieldMapperTests.java | 11 ++ .../mapper/GeoPointFieldMapperTests.java | 6 + .../mapper/GeoShapeFieldMapperTests.java | 6 + .../mapper/HalfFloatFieldMapperTests.java | 13 ++ .../index/mapper/IntegerFieldMapperTests.java | 11 ++ .../index/mapper/IpFieldMapperTests.java | 6 + .../index/mapper/KeywordFieldMapperTests.java | 37 +++--- .../LegacyGeoShapeFieldMapperTests.java | 6 + .../index/mapper/LongFieldMapperTests.java | 17 +++ .../index/mapper/NumberFieldMapperTests.java | 9 +- .../index/mapper/RangeFieldMapperTests.java | 9 ++ .../index/mapper/ShortFieldMapperTests.java | 11 ++ .../index/mapper/TextFieldMapperTests.java | 11 ++ .../flattened/FlattenedFieldMapperTests.java | 7 + .../AbstractNumericFieldMapperTestCase.java | 7 + .../index/mapper/MapperTestCase.java | 120 ++++++++++++++++-- .../mapper/HistogramFieldMapperTests.java | 7 + ...AggregateDoubleMetricFieldMapperTests.java | 6 + .../ConstantKeywordFieldMapperTests.java | 10 ++ .../UnsignedLongFieldMapperTests.java | 23 ++++ .../VersionStringFieldMapperTests.java | 22 ++++ ...GeoShapeWithDocValuesFieldMapperTests.java | 6 + .../index/mapper/PointFieldMapperTests.java | 6 + .../index/mapper/ShapeFieldMapperTests.java | 7 + .../mapper/DenseVectorFieldMapperTests.java | 7 + .../mapper/WildcardFieldMapperTests.java | 5 + 42 files changed, 544 insertions(+), 36 deletions(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index 29941feeb647e..27e13c4e6b60f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -136,4 +136,10 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep assertEquals("[rank_feature] fields do not support indexing multiple values for the same field [foo.field] in the same document", e.getCause().getMessage()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java index ea33861ac2135..5a8463c26347d 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapperTests.java @@ -135,4 +135,10 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep assertEquals("[rank_features] fields do not support indexing multiple values for the same rank feature [foo.field.bar] in " + "the same document", e.getCause().getMessage()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index 96ba6b7cd4655..39fd6333b1dbd 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -271,4 +271,30 @@ public void testRejectIndexOptions() { containsString("Failed to parse mapping: unknown parameter [index_options] on mapper [field] of type [scaled_float]")); } + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + // Large floats are a terrible idea but the round trip should still work no matter how badly you configure the field + b.field("type", "scaled_float").field("scaling_factor", randomDoubleBetween(0, Float.MAX_VALUE, true)); + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + /* + * randomDoubleBetween will smear the random values out across a huge + * range of valid values. + */ + double v = randomDoubleBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true); + switch (between(0, 3)) { + case 0: + return v; + case 1: + return (float) v; + case 2: + return Double.toString(v); + case 3: + return Float.toString((float) v); + default: + throw new IllegalArgumentException(); + } + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 5a6acbfc71f64..c48deb423c05a 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -745,4 +745,10 @@ private static PrefixFieldMapper getPrefixFieldMapper(DocumentMapper defaultMapp assertThat(mapper, instanceOf(PrefixFieldMapper.class)); return (PrefixFieldMapper) mapper; } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We don't have doc values or fielddata", true); + return null; + } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java index f5167c924186d..1af19b60d27e1 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/TokenCountFieldMapperTests.java @@ -177,4 +177,20 @@ private ParseContext.Document parseDocument(DocumentMapper mapper, SourceToParse return mapper.parse(request) .docs().stream().findFirst().orElseThrow(() -> new IllegalStateException("Test object not parsed")); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + int words = between(1, 1000); + StringBuilder b = new StringBuilder(words * 5); + b.append(randomAlphaOfLength(4)); + for (int w = 1; w < words; w++) { + b.append(' ').append(randomAlphaOfLength(4)); + } + return b.toString(); + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", "token_count").field("analyzer", "standard"); + } } diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index aef9fa2b53f04..a0de295102d45 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -136,7 +136,7 @@ public Query regexpQuery(String value, int syntaxFlags, int matchFlags, int maxD throw new UnsupportedOperationException("[regexp] queries are not supported on [" + CONTENT_TYPE + "] fields."); } - public static DocValueFormat COLLATE_FORMAT = new DocValueFormat() { + public static final DocValueFormat COLLATE_FORMAT = new DocValueFormat() { @Override public String getWriteableName() { return "collate"; diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index 6a0171760b714..a71ecf1b65aef 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -10,6 +10,7 @@ import com.ibm.icu.text.Collator; import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.util.ULocale; + import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -280,4 +281,13 @@ public void testUpdateIgnoreAbove() throws IOException { assertEquals(0, fields.length); } + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + assumeFalse("docvalue_fields is broken", true); + // https://github.com/elastic/elasticsearch/issues/70276 + /* + * docvalue_fields loads garbage bytes. + */ + return null; + } } diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index afb439c59cb68..619858ce4bcfe 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.analysis.StandardTokenizerFactory; import org.elasticsearch.index.analysis.TokenFilterFactory; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; @@ -552,4 +553,9 @@ public void testAnalyzedFieldPositionIncrementWithoutPositions() { } } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("annotated_text doesn't have fielddata so we can't check against anything here.", true); + return null; + } } diff --git a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java index 2a30253a2c2a3..962c2e7c0cd9f 100644 --- a/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java +++ b/plugins/mapper-murmur3/src/test/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapperTests.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.IndexableField; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.plugin.mapper.MapperMurmur3Plugin; @@ -56,4 +57,9 @@ public void testDefaults() throws Exception { assertEquals(DocValuesType.SORTED_NUMERIC, field.fieldType().docValuesType()); } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java index d68a0b08c4730..bcf2a5b3af9d0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeType.java @@ -546,6 +546,10 @@ public Query intersectsQuery(String field, Object from, Object to, boolean inclu } }; + public final String name; + private final NumberFieldMapper.NumberType numberType; + public final LengthType lengthType; + RangeType(String name, LengthType lengthType) { this.name = name; this.numberType = null; @@ -699,9 +703,9 @@ public final Mapper.TypeParser parser() { return new FieldMapper.TypeParser((n, c) -> new RangeFieldMapper.Builder(n, this, c.getSettings())); } - public final String name; - private final NumberFieldMapper.NumberType numberType; - public final LengthType lengthType; + NumberFieldMapper.NumberType numberType() { + return numberType; + } public enum LengthType { FIXED_4 { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java index b5cfdebc67c6e..0cbe10cff3c55 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BinaryFieldMapperTests.java @@ -118,4 +118,16 @@ public void testStoredValue() throws IOException { assertEquals(new BytesArray(value), originalValue); } } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We can't parse the binary doc values we send", true); + // AwaitsFix https://github.com/elastic/elasticsearch/issues/70244 + return null; + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", "binary").field("doc_values", true); // enable doc_values so the test is happy + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 08b4cb3f868c6..5ef540dfba0bc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -162,4 +162,20 @@ public void testDocValues() throws Exception { assertEquals(DocValuesType.NONE, fields[0].fieldType().docValuesType()); assertEquals(DocValuesType.SORTED_NUMERIC, fields[1].fieldType().docValuesType()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + switch (between(0, 3)) { + case 0: + return randomBoolean(); + case 1: + return randomBoolean() ? "true" : "false"; + case 2: + return randomBoolean() ? "true" : ""; + case 3: + return randomBoolean() ? "true" : null; + default: + throw new IllegalStateException(); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java index ccb8fa12c0d30..78e4555eb98a6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ByteFieldMapperTests.java @@ -35,4 +35,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "byte"); } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomByte(); + } + if (randomBoolean()) { + return randomDouble(); + } + return randomDoubleBetween(Byte.MIN_VALUE, Byte.MAX_VALUE, true); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index 784120aae828a..bfaec5431aa51 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -778,4 +778,9 @@ protected V featureValueOf(T actual) { }; } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We don't have doc values or fielddata", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index 17e1772d607f8..8bb0a27ea7162 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.search.DocValueFormat; @@ -500,6 +501,50 @@ public void testFetchNanosFromFixedNanosFormatted() throws IOException { ); } + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", randomBoolean() ? "date" : "date_nanos"); + } + + @Override + protected String randomFetchTestFormat() { + // TODO more choices! The test should work fine even for choices that throw out a ton of precision. + switch (randomInt(2)) { + case 0: + return null; + case 1: + return "epoch_millis"; + case 2: + return "iso8601"; + default: + throw new IllegalStateException(); + } + } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + switch (((DateFieldType) ft).resolution()) { + case MILLISECONDS: + if (randomBoolean()) { + return randomIs8601Nanos(MAX_ISO_DATE); + } + return randomLongBetween(0, Long.MAX_VALUE); + case NANOSECONDS: + switch (randomInt(2)) { + case 0: + return randomLongBetween(0, MAX_NANOS); + case 1: + return randomIs8601Nanos(MAX_NANOS); + case 2: + return new BigDecimal(randomDecimalNanos(MAX_MILLIS_DOUBLE_NANOS_KEEPS_PRECISION)); + default: + throw new IllegalStateException(); + } + default: + throw new IllegalStateException(); + } + } + private MapperService dateNanosMapperService() throws IOException { return createMapperService(mapping(b -> b.startObject("field").field("type", "date_nanos").endObject())); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java index 8425d18adfa4a..4b16565c45866 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldMapperTests.java @@ -36,4 +36,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "double"); } + + @Override + protected Number randomNumber() { + /* + * The source parser and doc values round trip will both increase + * the precision to 64 bits if the value is less precise. + * randomDoubleBetween will smear the values out across a wide + * range of valid values. + */ + return randomBoolean() ? randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true) : randomFloat(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java index 3727bf683e6d4..7a44465c805a9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FloatFieldMapperTests.java @@ -36,4 +36,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "float"); } + + @Override + protected Number randomNumber() { + /* + * The source parser and doc values round trip will both reduce + * the precision to 32 bits if the value is more precise. + * randomDoubleBetween will smear the values out across a wide + * range of valid values. + */ + return randomBoolean() ? randomDoubleBetween(-Float.MAX_VALUE, Float.MAX_VALUE, true) : randomFloat(); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index a11e608b68186..b7d129d01170e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -340,4 +340,10 @@ protected void assertSearchable(MappedFieldType fieldType) { //always searchable even if it uses TextSearchInfo.NONE assertTrue(fieldType.isSearchable()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 7ce12802938c3..1c18bbd637ac6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -226,4 +226,10 @@ protected void assertSearchable(MappedFieldType fieldType) { //always searchable even if it uses TextSearchInfo.NONE assertTrue(fieldType.isSearchable()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java index 08f198513f688..9006337f89b19 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatFieldMapperTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.HalfFloatPoint; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; @@ -38,4 +39,16 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "half_float"); } + + @Override + protected Number randomNumber() { + /* + * The native valueFetcher returns 32 bits of precision but the + * doc values fetcher returns 16 bits of precision. To make it + * all line up we round here instead of in the fetcher. This bug + * is tracked in: + * https://github.com/elastic/elasticsearch/issues/70260 + */ + return HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(randomFloat())); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java index af9ea505b3361..3df1442232eb1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IntegerFieldMapperTests.java @@ -36,4 +36,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "integer"); } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomInt(); + } + if (randomBoolean()) { + return randomDouble(); + } + return randomDoubleBetween(Integer.MIN_VALUE, Integer.MAX_VALUE, true); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index 9e49fabd3c4e8..70b45169ff3d1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -17,6 +17,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.termvectors.TermVectorsService; @@ -202,4 +203,9 @@ public void testNullValue() throws IOException { })); assertWarnings("Error parsing [:1] as IP in [null_value] on field [field]); [null_value] will be ignored"); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + return NetworkAddress.format(randomIp(randomBoolean())); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 2f41579d73b85..f6ef9a75df139 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -40,14 +40,11 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; -import static java.util.stream.Collectors.toList; import static org.apache.lucene.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -462,25 +459,21 @@ public void testSplitQueriesOnWhitespace() throws IOException { ); } - public void testFetch() throws IOException { - assertFetch(keywordMapperService(), "field", randomAlphaOfLength(5), null); - } - - public void testFetchMany() throws IOException { - /* - * When we have many values doc values will sort and unique them. - * Source fetching won't, but we expect that. So we test with - * sorted and uniqued values. - */ - int count = between(2, 10); - Set values = new HashSet<>(); - while (values.size() < count) { - values.add(randomAlphaOfLength(5)); + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + switch (between(0, 3)) { + case 0: + return randomAlphaOfLengthBetween(1, 100); + case 1: + return randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100); + case 2: + return randomLong(); + case 3: + return randomDouble(); + case 4: + return randomBoolean(); + default: + throw new IllegalStateException(); } - assertFetch(keywordMapperService(), "field", values.stream().sorted().collect(toList()), null); - } - - private MapperService keywordMapperService() throws IOException { - return createMapperService(mapping(b -> b.startObject("field").field("type", "keyword").endObject())); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java index fd04a0ed1203c..552d8d81cc383 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -634,4 +634,10 @@ protected void assertSearchable(MappedFieldType fieldType) { //always searchable even if it uses TextSearchInfo.NONE assertTrue(fieldType.isSearchable()); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java index 2c45d38dd8124..7a05914324497 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldMapperTests.java @@ -59,4 +59,21 @@ public void testLongIndexingCoercesIntoRange() throws Exception { doc = mapper.parse(source(b -> b.field("field", "-9223372036854775808.9"))); assertThat(doc.rootDoc().getFields("field"), arrayWithSize(2)); } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomLong(); + } + if (randomBoolean()) { + return randomDouble(); + } + assumeFalse("https://github.com/elastic/elasticsearch/issues/70585", true); + return randomDoubleBetween(Long.MIN_VALUE, Long.MAX_VALUE, true); + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/70585") + public void testFetchCoerced() throws IOException { + assertFetch(randomFetchTestMapper(), "field", 3.783147882954537E18, randomFetchTestFormat()); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 1f0babb1732d5..6bce801ce7484 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -205,7 +205,7 @@ protected void testNullValue() throws IOException { assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType()); assertFalse(dvField.fieldType().stored()); } - + public void testOutOfRangeValues() throws IOException { for(OutOfRangeSpec item : outOfRangeSpecs()) { @@ -218,6 +218,13 @@ public void testOutOfRangeValues() throws IOException { e.getCause().getMessage(), containsString(item.message)); } } + } + @Override + protected final Object generateRandomInputValue(MappedFieldType ft) { + Number n = randomNumber(); + return randomBoolean() ? n : n.toString(); } + + protected abstract Number randomNumber(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index fde4cb56adc78..aeda7885ce024 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -346,4 +346,13 @@ public void testIllegalFormatField() throws Exception { ); assertThat(e.getMessage(), containsString("Invalid format: [[test_format]]: Unknown pattern letter: t")); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + // Doc value fetching crashes. + // https://github.com/elastic/elasticsearch/issues/70269 + // TODO when we fix doc values fetcher we should add tests for date and ip ranges. + assumeFalse("DocValuesFetcher doesn't work", true); + return null; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java index 44ab5d4f70b23..a78e6234b7877 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ShortFieldMapperTests.java @@ -36,4 +36,15 @@ protected List outOfRangeSpecs() { protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "short"); } + + @Override + protected Number randomNumber() { + if (randomBoolean()) { + return randomShort(); + } + if (randomBoolean()) { + return randomDouble(); + } + return randomDoubleBetween(Short.MIN_VALUE, Short.MAX_VALUE, true); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 5a7388a66a5a3..c7ee493dbf7aa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -1079,4 +1079,15 @@ public void testSimpleMerge() throws IOException { assertThat(mapperService.documentMapper().mappers().getMapper("field"), instanceOf(TextFieldMapper.class)); assertThat(mapperService.documentMapper().mappers().getMapper("other_field"), instanceOf(KeywordFieldMapper.class)); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("We don't have a way to assert things here", true); + return null; + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + assumeFalse("We don't have a way to assert things here", true); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 1260b59f532b2..fbfbc0f1a2175 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; @@ -333,4 +334,10 @@ public void testSplitQueriesOnWhitespace() throws IOException { assertTokenStreamContents(keyedFieldType.getTextSearchInfo().getSearchAnalyzer().analyzer().tokenStream("", "Hello World"), new String[] {"Hello", "World"}); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java index 08a4e322e3174..dd6eb95a9d7ce 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/AbstractNumericFieldMapperTestCase.java @@ -7,6 +7,8 @@ */ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.xcontent.XContentBuilder; + import java.io.IOException; import java.util.Set; @@ -77,4 +79,9 @@ public final void testNullValue() throws IOException { } protected abstract void doTestNullValue(String type) throws IOException; + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", randomFrom(types())); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index ce9b362494b59..9ba70e61035de 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -46,7 +46,9 @@ import java.util.function.Consumer; import java.util.function.Supplier; +import static java.util.stream.Collectors.toList; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Mockito.mock; @@ -298,16 +300,11 @@ public final void testDeprecatedBoost() throws IOException { protected final List fetchFromDocValues(MapperService mapperService, MappedFieldType ft, DocValueFormat format, Object sourceValue) throws IOException { - BiFunction, IndexFieldData> fieldDataLookup = (mft, lookupSource) -> mft - .fielddataBuilder("test", () -> { - throw new UnsupportedOperationException(); - }) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); SetOnce> result = new SetOnce<>(); withLuceneIndex(mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft)); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); @@ -457,6 +454,82 @@ protected void assertSearchable(MappedFieldType fieldType) { assertEquals(fieldType.isSearchable(), fieldType.getTextSearchInfo() != TextSearchInfo.NONE); } + /** + * Asserts that fetching a single value from doc values and from the native + * {@link MappedFieldType#valueFetcher} produce the same results. + *

+ * Generally this method covers many many random cases but rarely. So if + * it fails its generally a good idea to capture its randomized + * parameters into a new method so we can be sure we consistently test + * any unique and interesting failure case. See the tests for + * {@link DateFieldMapper} for some examples. + */ + public final void testFetch() throws IOException { + MapperService mapperService = randomFetchTestMapper(); + try { + MappedFieldType ft = mapperService.fieldType("field"); + assertFetch(mapperService, "field", generateRandomInputValue(ft), randomFetchTestFormat()); + } finally { + assertParseMinimalWarnings(); + } + } + + /** + * Asserts that fetching many values from doc values and from the native + * {@link MappedFieldType#valueFetcher} produce the same results. + *

+ * Generally this method covers many many random cases but rarely. So if + * it fails its generally a good idea to capture its randomized + * parameters into a new method so we can be sure we consistently test + * any unique and interesting failure case. See the tests for + * {@link DateFieldMapper} for some examples. + */ + public final void testFetchMany() throws IOException { + MapperService mapperService = randomFetchTestMapper(); + try { + MappedFieldType ft = mapperService.fieldType("field"); + int count = between(2, 10); + List values = new ArrayList<>(count); + while (values.size() < count) { + values.add(generateRandomInputValue(ft)); + } + assertFetch(mapperService, "field", values, randomFetchTestFormat()); + } finally { + assertParseMinimalWarnings(); + } + } + + protected final MapperService randomFetchTestMapper() throws IOException { + return createMapperService(mapping(b -> { + b.startObject("field"); + randomFetchTestFieldConfig(b); + b.endObject(); + })); + } + + /** + * Field configuration for {@link #testFetch} and {@link #testFetchMany}. + * Default implementation delegates to {@link #minimalMapping} but can + * be overridden to randomize the field type and options. + */ + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + minimalMapping(b); + } + + /** + * A random format to use when tripping in {@link #testFetch} and + * {@link #testFetchMany}. + */ + protected String randomFetchTestFormat() { + return null; + } + + /** + * Create a random {@code _source} value for this field. Must be compatible + * with {@link XContentBuilder#value(Object)} and the field's parser. + */ + protected abstract Object generateRandomInputValue(MappedFieldType ft); + /** * Assert that fetching a value using {@link MappedFieldType#valueFetcher} * produces the same value as fetching using doc values. @@ -470,15 +543,44 @@ protected void assertFetch(MapperService mapperService, String field, Object val ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); + when(searchExecutionContext.getForField(ft)).thenAnswer( + inv -> { return fieldDataLookup().apply(ft, () -> { throw new UnsupportedOperationException(); }); } + ); ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); withLuceneIndex(mapperService, iw -> iw.addDocuments(doc.docs()), ir -> { SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSegmentAndDocument(ir.leaves().get(0), 0); docValueFetcher.setNextReader(ir.leaves().get(0)); - List fromDocValues = docValueFetcher.fetchValues(sourceLookup); - List fromNative = nativeFetcher.fetchValues(sourceLookup); - assertEquals(fromDocValues, fromNative); + nativeFetcher.setNextReader(ir.leaves().get(0)); + List fromDocValues = docValueFetcher.fetchValues(sourceLookup); + List fromNative = nativeFetcher.fetchValues(sourceLookup); + /* + * The native fetcher uses byte, short, etc but doc values always + * uses long or double. This difference is fine because on the outside + * users can't see it. + */ + fromNative = fromNative.stream().map(o -> { + if (o instanceof Integer || o instanceof Short || o instanceof Byte) { + return ((Number) o).longValue(); + } + if (o instanceof Float) { + return ((Float) o).doubleValue(); + } + return o; + }).collect(toList()); + /* + * Doc values sort according to something appropriate to the field + * and the native fetchers usually don't sort. We're ok with this + * difference. But we have to convince the test we're ok with it. + */ + assertThat("fetching " + value, fromNative, containsInAnyOrder(fromDocValues.toArray())); }); } + + private BiFunction, IndexFieldData> fieldDataLookup() { + return (mft, lookupSource) -> mft + .fielddataBuilder("test", lookupSource) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + } } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java index 2ef401206ef04..804cd322ab370 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapperTests.java @@ -8,6 +8,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; @@ -293,4 +294,10 @@ public void testNegativeCount() throws Exception { Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source)); assertThat(e.getCause().getMessage(), containsString("[counts] elements must be >= 0 but got -3")); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java index 6812bd6cce77b..32919a284d33e 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapperTests.java @@ -528,4 +528,10 @@ protected void assertExistsQuery(MappedFieldType fieldType, Query query, ParseCo assertDocValuesField(fields, "field." + defaultMetric); assertNoFieldNamesField(fields); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 8291d15887c95..fd2ed68314fd5 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/internalClusterTest/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -162,4 +162,14 @@ protected void registerParameters(ParameterChecker checker) throws IOException { b.field("value", "bar"); })); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + return ((ConstantKeywordFieldType) ft).value(); + } + + @Override + protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException { + b.field("type", "constant_keyword").field("value", randomAlphaOfLengthBetween(1, 10)); + } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 1afb898a59eb4..0931ce4ef11dd 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperTestCase; @@ -216,4 +217,26 @@ public void testExistsQueryDocValuesDisabled() throws IOException { assertParseMinimalWarnings(); } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + Number n = randomNumericValue(); + return randomBoolean() ? n : n.toString(); + } + + private Number randomNumericValue() { + switch (randomInt(8)) { + case 0: + return randomNonNegativeByte(); + case 1: + return (short) between(0, Short.MAX_VALUE); + case 2: + return randomInt(Integer.MAX_VALUE); + case 3: + case 4: + return randomNonNegativeLong(); + default: + BigInteger big = BigInteger.valueOf(randomLongBetween(0, Long.MAX_VALUE)).shiftLeft(1); + return big.add(randomBoolean() ? BigInteger.ONE : BigInteger.ZERO); + } + } } diff --git a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java index 707b1bdea948a..2095bb5d526b9 100644 --- a/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java +++ b/x-pack/plugin/mapper-version/src/test/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapperTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; @@ -128,4 +129,25 @@ public void testFailsParsingNestedList() throws IOException { ex.getMessage() ); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + return randomVersionNumber() + (randomBoolean() ? "" : randomPrerelease()); + } + + private String randomVersionNumber() { + int numbers = between(1, 3); + String v = Integer.toString(between(0, 100)); + for (int i = 1; i < numbers; i++) { + v += "." + between(0, 100); + } + return v; + } + + private String randomPrerelease() { + if (rarely()) { + return randomFrom("alpha", "beta", "prerelease", "whatever"); + } + return randomFrom("alpha", "beta", "") + randomVersionNumber(); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index 65b4661410d19..226af3b63b2ad 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -323,4 +323,10 @@ public String toXContentString(GeoShapeWithDocValuesFieldMapper mapper) throws I protected void assertSearchable(MappedFieldType fieldType) { } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index 1b5e055bbdfb1..3df7552ed9f91 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; @@ -211,4 +212,9 @@ public void testIgnoreZValue() throws IOException { } } + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 240835263ac82..ed91cc2652e4b 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; @@ -235,4 +236,10 @@ public String toXContentString(ShapeFieldMapper mapper, boolean includeDefaults) public String toXContentString(ShapeFieldMapper mapper) { return toXContentString(mapper, true); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java index 5746cff88446a..f5a33dde9ebc6 100644 --- a/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java +++ b/x-pack/plugin/vectors/src/test/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapperTests.java @@ -13,6 +13,7 @@ import org.elasticsearch.Version; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperTestCase; import org.elasticsearch.index.mapper.ParsedDocument; @@ -153,4 +154,10 @@ public void testDocumentsWithIncorrectDims() throws Exception { () -> mapper.parse(source(b -> b.array("field", invalidVector2)))); assertThat(e2.getCause().getMessage(), containsString("has number of dimensions [2] less than defined in the mapping [3]")); } + + @Override + protected Object generateRandomInputValue(MappedFieldType ft) { + assumeFalse("Test implemented in a follow up", true); + return null; + } } diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index f5d0fe89d691f..c9aa8da3af98c 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -984,4 +984,9 @@ private String getRandomWildcardPattern() { } return sb.toString(); } + + @Override + protected String generateRandomInputValue(MappedFieldType ft) { + return randomAlphaOfLengthBetween(1, 100); + } }