diff --git a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index a21b53cdf51bf..63eff82ddb0ce 100644 --- a/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -105,6 +105,7 @@ public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFi this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields(); this.indicesQueriesRegistry = indicesQueriesRegistry; this.percolatorQueryCache = percolatorQueryCache; + this.nestedScope = new NestedScope(); } public QueryShardContext(QueryShardContext source) { @@ -113,6 +114,7 @@ public QueryShardContext(QueryShardContext source) { } + @Override public QueryShardContext clone() { return new QueryShardContext(indexSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry, percolatorQueryCache); } diff --git a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index a5707ea4a537f..02f39cf94903c 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; @@ -27,8 +28,14 @@ import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.search.MultiValueMode; import java.io.IOException; import java.util.Objects; @@ -47,6 +54,13 @@ public class FieldSortBuilder extends SortBuilder implements S public static final ParseField SORT_MODE = new ParseField("mode"); public static final ParseField UNMAPPED_TYPE = new ParseField("unmapped_type"); + /** + * special field name to sort by index order + */ + public static final String DOC_FIELD_NAME = "_doc"; + private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC); + private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true); + private final String fieldName; private Object missing; @@ -161,7 +175,7 @@ public SortMode sortMode() { * TODO should the above getters and setters be deprecated/ changed in * favour of real getters and setters? */ - public FieldSortBuilder setNestedFilter(QueryBuilder nestedFilter) { + public FieldSortBuilder setNestedFilter(QueryBuilder nestedFilter) { this.nestedFilter = nestedFilter; return this; } @@ -170,7 +184,7 @@ public FieldSortBuilder setNestedFilter(QueryBuilder nestedFilter) { * Returns the nested filter that the nested objects should match with in * order to be taken into account for sorting. */ - public QueryBuilder getNestedFilter() { + public QueryBuilder getNestedFilter() { return this.nestedFilter; } @@ -219,6 +233,49 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder; } + @Override + public SortField build(QueryShardContext context) throws IOException { + if (DOC_FIELD_NAME.equals(fieldName)) { + if (order == SortOrder.DESC) { + return SORT_DOC_REVERSE; + } else { + return SORT_DOC; + } + } else { + MappedFieldType fieldType = context.fieldMapper(fieldName); + if (fieldType == null) { + if (unmappedType != null) { + fieldType = context.getMapperService().unmappedFieldType(unmappedType); + } else { + throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on"); + } + } + + if (!fieldType.isSortable()) { + throw new QueryShardException(context, "Sorting not supported for field[" + fieldName + "]"); + } + + MultiValueMode localSortMode = null; + if (sortMode != null) { + localSortMode = MultiValueMode.fromString(sortMode.toString()); + } + + if (fieldType.isNumeric() == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { + throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); + } + + boolean reverse = (order == SortOrder.DESC); + if (localSortMode == null) { + localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN; + } + + final Nested nested = resolveNested(context, nestedPath, nestedFilter); + IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.getForField(fieldType) + .comparatorSource(missing, localSortMode, nested); + return new SortField(fieldType.name(), fieldComparatorSource, reverse); + } + } + @Override public boolean equals(Object other) { if (this == other) { diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 9785a0fc24048..4263e148323f5 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -19,8 +19,18 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BitSet; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.Version; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.geo.GeoDistance; +import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.io.stream.StreamInput; @@ -28,8 +38,17 @@ import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.NumericDoubleValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.MultiValueMode; import java.io.IOException; import java.util.ArrayList; @@ -45,6 +64,14 @@ public class GeoDistanceSortBuilder extends SortBuilder public static final String NAME = "_geo_distance"; public static final boolean DEFAULT_COERCE = false; public static final boolean DEFAULT_IGNORE_MALFORMED = false; + public static final ParseField UNIT_FIELD = new ParseField("unit"); + public static final ParseField REVERSE_FIELD = new ParseField("reverse"); + public static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type"); + public static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize"); + public static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed"); + public static final ParseField SORTMODE_FIELD = new ParseField("mode", "sort_mode"); + public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path"); + public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter"); static final GeoDistanceSortBuilder PROTOTYPE = new GeoDistanceSortBuilder("", -1, -1); @@ -280,22 +307,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.endArray(); - builder.field("unit", unit); - builder.field("distance_type", geoDistance.name().toLowerCase(Locale.ROOT)); + builder.field(UNIT_FIELD.getPreferredName(), unit); + builder.field(DISTANCE_TYPE_FIELD.getPreferredName(), geoDistance.name().toLowerCase(Locale.ROOT)); builder.field(ORDER_FIELD.getPreferredName(), order); if (sortMode != null) { - builder.field("mode", sortMode); + builder.field(SORTMODE_FIELD.getPreferredName(), sortMode); } if (nestedPath != null) { - builder.field("nested_path", nestedPath); + builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath); } if (nestedFilter != null) { - builder.field("nested_filter", nestedFilter, params); + builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, params); } - builder.field("coerce", coerce); - builder.field("ignore_malformed", ignoreMalformed); + builder.field(COERCE_FIELD.getPreferredName(), coerce); + builder.field(IGNORE_MALFORMED_FIELD.getPreferredName(), ignoreMalformed); builder.endObject(); return builder; @@ -383,6 +410,7 @@ public GeoDistanceSortBuilder readFrom(StreamInput in) throws IOException { @Override public GeoDistanceSortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException { XContentParser parser = context.parser(); + ParseFieldMatcher parseFieldMatcher = context.parseFieldMatcher(); String fieldName = null; List geoPoints = new ArrayList<>(); DistanceUnit unit = DistanceUnit.DEFAULT; @@ -405,40 +433,37 @@ public GeoDistanceSortBuilder fromXContent(QueryParseContext context, String ele fieldName = currentName; } else if (token == XContentParser.Token.START_OBJECT) { - // the json in the format of -> field : { lat : 30, lon : 12 } - if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) { - // TODO Note to remember: while this is kept as a QueryBuilder internally, - // we need to make sure to call toFilter() on it once on the shard - // (e.g. in the new build() method) + if (parseFieldMatcher.match(currentName, NESTED_FILTER_FIELD)) { nestedFilter = context.parseInnerQueryBuilder(); } else { + // the json in the format of -> field : { lat : 30, lon : 12 } fieldName = currentName; GeoPoint point = new GeoPoint(); GeoUtils.parseGeoPoint(parser, point); geoPoints.add(point); } } else if (token.isValue()) { - if ("reverse".equals(currentName)) { + if (parseFieldMatcher.match(currentName, REVERSE_FIELD)) { order = parser.booleanValue() ? SortOrder.DESC : SortOrder.ASC; - } else if ("order".equals(currentName)) { + } else if (parseFieldMatcher.match(currentName, ORDER_FIELD)) { order = SortOrder.fromString(parser.text()); - } else if ("unit".equals(currentName)) { + } else if (parseFieldMatcher.match(currentName, UNIT_FIELD)) { unit = DistanceUnit.fromString(parser.text()); - } else if ("distance_type".equals(currentName) || "distanceType".equals(currentName)) { + } else if (parseFieldMatcher.match(currentName, DISTANCE_TYPE_FIELD)) { geoDistance = GeoDistance.fromString(parser.text()); - } else if ("coerce".equals(currentName) || "normalize".equals(currentName)) { + } else if (parseFieldMatcher.match(currentName, COERCE_FIELD)) { coerce = parser.booleanValue(); if (coerce == true) { ignoreMalformed = true; } - } else if ("ignore_malformed".equals(currentName)) { + } else if (parseFieldMatcher.match(currentName, IGNORE_MALFORMED_FIELD)) { boolean ignore_malformed_value = parser.booleanValue(); if (coerce == false) { ignoreMalformed = ignore_malformed_value; } - } else if ("sort_mode".equals(currentName) || "sortMode".equals(currentName) || "mode".equals(currentName)) { + } else if (parseFieldMatcher.match(currentName, SORTMODE_FIELD)) { sortMode = SortMode.fromString(parser.text()); - } else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) { + } else if (parseFieldMatcher.match(currentName, NESTED_PATH_FIELD)) { nestedPath = parser.text(); } else { GeoPoint point = new GeoPoint(); @@ -461,7 +486,85 @@ public GeoDistanceSortBuilder fromXContent(QueryParseContext context, String ele result.coerce(coerce); result.ignoreMalformed(ignoreMalformed); return result; + } + + @Override + public SortField build(QueryShardContext context) throws IOException { + final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0); + // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes + List localPoints = new ArrayList(); + for (GeoPoint geoPoint : this.points) { + localPoints.add(new GeoPoint(geoPoint)); + } + + if (!indexCreatedBeforeV2_0 && !ignoreMalformed) { + for (GeoPoint point : localPoints) { + if (GeoUtils.isValidLatitude(point.lat()) == false) { + throw new ElasticsearchParseException("illegal latitude value [{}] for [GeoDistanceSort]", point.lat()); + } + if (GeoUtils.isValidLongitude(point.lon()) == false) { + throw new ElasticsearchParseException("illegal longitude value [{}] for [GeoDistanceSort]", point.lon()); + } + } + } + + if (coerce) { + for (GeoPoint point : localPoints) { + GeoUtils.normalizePoint(point, coerce, coerce); + } + } + + boolean reverse = (order == SortOrder.DESC); + final MultiValueMode finalSortMode; + if (sortMode == null) { + finalSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN; + } else { + finalSortMode = MultiValueMode.fromString(sortMode.toString()); + } + + MappedFieldType fieldType = context.fieldMapper(fieldName); + if (fieldType == null) { + throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort"); + } + final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType); + final FixedSourceDistance[] distances = new FixedSourceDistance[localPoints.size()]; + for (int i = 0; i< localPoints.size(); i++) { + distances[i] = geoDistance.fixedSourceDistance(localPoints.get(i).lat(), localPoints.get(i).lon(), unit); + } + + final Nested nested = resolveNested(context, nestedPath, nestedFilter); + + IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() { + + @Override + public SortField.Type reducedType() { + return SortField.Type.DOUBLE; + } + + @Override + public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { + return new FieldComparator.DoubleComparator(numHits, null, null) { + @Override + protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException { + final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues(); + final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(geoPointValues, distances); + final NumericDoubleValues selectedValues; + if (nested == null) { + selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE); + } else { + final BitSet rootDocs = nested.rootDocs(context); + final DocIdSetIterator innerDocs = nested.innerDocs(context); + selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs, + context.reader().maxDoc()); + } + return selectedValues.getRawDoubleValues(); + } + }; + } + + }; + return new SortField(fieldName, geoDistanceComparatorSource, reverse); } static void parseGeoPoints(XContentParser parser, List geoPoints) throws IOException { diff --git a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java index d1eabf89e4590..aff0e68fc1d5c 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java +++ b/core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java @@ -62,7 +62,7 @@ public String[] names() { } @Override - public SortField parse(XContentParser parser, QueryShardContext context) throws Exception { + public SortField parse(XContentParser parser, QueryShardContext context) throws IOException { String fieldName = null; List geoPoints = new ArrayList<>(); DistanceUnit unit = DistanceUnit.DEFAULT; diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java index 76ca56f0f9f97..422be33978846 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.search.SortField; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParsingException; @@ -27,6 +28,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; import java.util.Objects; @@ -40,6 +42,8 @@ public class ScoreSortBuilder extends SortBuilder implements S static final ScoreSortBuilder PROTOTYPE = new ScoreSortBuilder(); public static final ParseField REVERSE_FIELD = new ParseField("reverse"); public static final ParseField ORDER_FIELD = new ParseField("order"); + private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE); + private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true); public ScoreSortBuilder() { // order defaults to desc when sorting on the _score @@ -84,6 +88,14 @@ public ScoreSortBuilder fromXContent(QueryParseContext context, String elementNa return result; } + public SortField build(QueryShardContext context) { + if (order == SortOrder.DESC) { + return SORT_SCORE; + } else { + return SORT_SCORE_REVERSE; + } + } + @Override public boolean equals(Object object) { if (this == object) { diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java index e77d12ce478a5..6005d9354ff6b 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortBuilder.java @@ -19,6 +19,12 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParsingException; @@ -27,14 +33,29 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.fielddata.NumericDoubleValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; +import org.elasticsearch.script.LeafSearchScript; import org.elasticsearch.script.Script; import org.elasticsearch.script.Script.ScriptField; +import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptParameterParser; import org.elasticsearch.script.ScriptParameterParser.ScriptParameterValue; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.MultiValueMode; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -56,7 +77,7 @@ public class ScriptSortBuilder extends SortBuilder implements private final Script script; - private ScriptSortType type; + private final ScriptSortType type; private SortMode sortMode; @@ -104,11 +125,15 @@ public ScriptSortType type() { } /** - * Defines which distance to use for sorting in the case a document contains multiple geo points. - * Possible values: min and max + * Defines which distance to use for sorting in the case a document contains multiple values.
+ * For {@link ScriptSortType#STRING}, the set of possible values is restricted to {@link SortMode#MIN} and {@link SortMode#MAX} */ public ScriptSortBuilder sortMode(SortMode sortMode) { Objects.requireNonNull(sortMode, "sort mode cannot be null."); + if (ScriptSortType.STRING.equals(type) && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || + sortMode == SortMode.MEDIAN)) { + throw new IllegalArgumentException("script sort of type [string] doesn't support mode [" + sortMode + "]"); + } this.sortMode = sortMode; return this; } @@ -244,6 +269,75 @@ public ScriptSortBuilder fromXContent(QueryParseContext context, String elementN return result; } + + @Override + public SortField build(QueryShardContext context) throws IOException { + final SearchScript searchScript = context.getScriptService().search( + context.lookup(), script, ScriptContext.Standard.SEARCH, Collections.emptyMap()); + + MultiValueMode valueMode = null; + if (sortMode != null) { + valueMode = MultiValueMode.fromString(sortMode.toString()); + } + boolean reverse = (order == SortOrder.DESC); + if (valueMode == null) { + valueMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN; + } + + final Nested nested = resolveNested(context, nestedPath, nestedFilter); + final IndexFieldData.XFieldComparatorSource fieldComparatorSource; + switch (type) { + case STRING: + fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, valueMode, nested) { + LeafSearchScript leafScript; + @Override + protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException { + leafScript = searchScript.getLeafSearchScript(context); + final BinaryDocValues values = new BinaryDocValues() { + final BytesRefBuilder spare = new BytesRefBuilder(); + @Override + public BytesRef get(int docID) { + leafScript.setDocument(docID); + spare.copyChars(leafScript.run().toString()); + return spare.get(); + } + }; + return FieldData.singleton(values, null); + } + @Override + protected void setScorer(Scorer scorer) { + leafScript.setScorer(scorer); + } + }; + break; + case NUMBER: + fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, valueMode, nested) { + LeafSearchScript leafScript; + @Override + protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws IOException { + leafScript = searchScript.getLeafSearchScript(context); + final NumericDoubleValues values = new NumericDoubleValues() { + @Override + public double get(int docID) { + leafScript.setDocument(docID); + return leafScript.runAsDouble(); + } + }; + return FieldData.singleton(values, null); + } + @Override + protected void setScorer(Scorer scorer) { + leafScript.setScorer(scorer); + } + }; + break; + default: + throw new QueryShardException(context, "custom script sort type [" + type + "] not supported"); + } + + return new SortField("_script", fieldComparatorSource, reverse); + } + @Override public boolean equals(Object object) { if (this == object) { diff --git a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java index c238ad6ccaf2d..af8531da87ce9 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java +++ b/core/src/main/java/org/elasticsearch/search/sort/ScriptSortParser.java @@ -66,7 +66,7 @@ public String[] names() { } @Override - public SortField parse(XContentParser parser, QueryShardContext context) throws Exception { + public SortField parse(XContentParser parser, QueryShardContext context) throws IOException { ScriptParameterParser scriptParameterParser = new ScriptParameterParser(); Script script = null; ScriptSortType type = null; @@ -140,7 +140,6 @@ public SortField parse(XContentParser parser, QueryShardContext context) throws sortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN; } - // If nested_path is specified, then wrap the `fieldComparatorSource` in a `NestedFieldComparatorSource` final Nested nested; if (nestedHelper != null && nestedHelper.getPath() != null) { BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); @@ -182,7 +181,6 @@ protected void setScorer(Scorer scorer) { }; break; case NUMBER: - // TODO: should we rather sort missing values last? fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, sortMode, nested) { LeafSearchScript leafScript; @Override diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java index 7852af4e97ed5..35d59de011c25 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilder.java @@ -19,12 +19,21 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.join.BitSetProducer; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; +import org.elasticsearch.index.mapper.object.ObjectMapper; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; +import java.io.IOException; import java.util.Objects; /** @@ -32,6 +41,30 @@ */ public abstract class SortBuilder> implements ToXContent { + protected static Nested resolveNested(QueryShardContext context, String nestedPath, QueryBuilder nestedFilter) throws IOException { + Nested nested = null; + if (nestedPath != null) { + BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); + ObjectMapper nestedObjectMapper = context.getObjectMapper(nestedPath); + if (nestedObjectMapper == null) { + throw new QueryShardException(context, "[nested] failed to find nested object under path [" + nestedPath + "]"); + } + if (!nestedObjectMapper.nested().isNested()) { + throw new QueryShardException(context, "[nested] nested object under path [" + nestedPath + "] is not of nested type"); + } + Query innerDocumentsQuery; + if (nestedFilter != null) { + context.nestedScope().nextLevel(nestedObjectMapper); + innerDocumentsQuery = QueryBuilder.rewriteQuery(nestedFilter, context).toFilter(context); + context.nestedScope().previousLevel(); + } else { + innerDocumentsQuery = nestedObjectMapper.nestedTypeFilter(); + } + nested = new Nested(rootDocumentsFilter, innerDocumentsQuery); + } + return nested; + } + protected SortOrder order = SortOrder.ASC; public static final ParseField ORDER_FIELD = new ParseField("order"); diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortBuilderParser.java b/core/src/main/java/org/elasticsearch/search/sort/SortBuilderParser.java index 90d54a501215b..706fa7863f053 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortBuilderParser.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortBuilderParser.java @@ -19,9 +19,11 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.search.SortField; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; import java.io.IOException; @@ -36,5 +38,10 @@ public interface SortBuilderParser extends NamedWriteable< * call * @return the new item */ - SortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException; + T fromXContent(QueryParseContext context, String elementName) throws IOException; + + /** + * Create a @link {@link SortField} from this builder. + */ + SortField build(QueryShardContext context) throws IOException; } diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java b/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java index fe0b62022fe6f..1ed2a457a5f96 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortParseElement.java @@ -30,10 +30,11 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.support.NestedInnerQueryParseSupport; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.SearchParseElement; -import org.elasticsearch.search.SearchParseException; import org.elasticsearch.search.internal.SearchContext; import java.io.IOException; @@ -49,7 +50,7 @@ */ public class SortParseElement implements SearchParseElement { - public static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE); + private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE); private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true); private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC); private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true); @@ -75,7 +76,28 @@ private static void addParser(Map parsers, SortParser parser } @Override - public void parse(XContentParser parser, SearchContext context) throws Exception { + public void parse(XContentParser parser, SearchContext context) throws IOException { + List sortFields = parse(parser, context.getQueryShardContext()); + if (!sortFields.isEmpty()) { + // optimize if we just sort on score non reversed, we don't really need sorting + boolean sort; + if (sortFields.size() > 1) { + sort = true; + } else { + SortField sortField = sortFields.get(0); + if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) { + sort = false; + } else { + sort = true; + } + } + if (sort) { + context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); + } + } + } + + List parse(XContentParser parser, QueryShardContext context) throws IOException { XContentParser.Token token = parser.currentToken(); List sortFields = new ArrayList<>(2); if (token == XContentParser.Token.START_ARRAY) { @@ -95,26 +117,10 @@ public void parse(XContentParser parser, SearchContext context) throws Exception } else { throw new IllegalArgumentException("malformed sort format, either start with array, object, or an actual string"); } - if (!sortFields.isEmpty()) { - // optimize if we just sort on score non reversed, we don't really need sorting - boolean sort; - if (sortFields.size() > 1) { - sort = true; - } else { - SortField sortField = sortFields.get(0); - if (sortField.getType() == SortField.Type.SCORE && !sortField.getReverse()) { - sort = false; - } else { - sort = true; - } - } - if (sort) { - context.sort(new Sort(sortFields.toArray(new SortField[sortFields.size()]))); - } - } + return sortFields; } - private void addCompoundSortField(XContentParser parser, SearchContext context, List sortFields) throws Exception { + private void addCompoundSortField(XContentParser parser, QueryShardContext context, List sortFields) throws IOException { XContentParser.Token token; while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -138,7 +144,7 @@ private void addCompoundSortField(XContentParser parser, SearchContext context, addSortField(context, sortFields, fieldName, reverse, unmappedType, missing, sortMode, nestedFilterParseHelper); } else { if (PARSERS.containsKey(fieldName)) { - sortFields.add(PARSERS.get(fieldName).parse(parser, context.getQueryShardContext())); + sortFields.add(PARSERS.get(fieldName).parse(parser, context)); } else { while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { if (token == XContentParser.Token.FIELD_NAME) { @@ -160,7 +166,7 @@ private void addCompoundSortField(XContentParser parser, SearchContext context, sortMode = MultiValueMode.fromString(parser.text()); } else if ("nested_path".equals(innerJsonName) || "nestedPath".equals(innerJsonName)) { if (nestedFilterParseHelper == null) { - nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context.getQueryShardContext()); + nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context); } nestedFilterParseHelper.setPath(parser.text()); } else { @@ -169,7 +175,7 @@ private void addCompoundSortField(XContentParser parser, SearchContext context, } else if (token == XContentParser.Token.START_OBJECT) { if ("nested_filter".equals(innerJsonName) || "nestedFilter".equals(innerJsonName)) { if (nestedFilterParseHelper == null) { - nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context.getQueryShardContext()); + nestedFilterParseHelper = new NestedInnerQueryParseSupport(parser, context); } nestedFilterParseHelper.filter(); } else { @@ -184,7 +190,7 @@ private void addCompoundSortField(XContentParser parser, SearchContext context, } } - private void addSortField(SearchContext context, List sortFields, String fieldName, boolean reverse, String unmappedType, @Nullable final String missing, MultiValueMode sortMode, NestedInnerQueryParseSupport nestedHelper) throws IOException { + private void addSortField(QueryShardContext context, List sortFields, String fieldName, boolean reverse, String unmappedType, @Nullable final String missing, MultiValueMode sortMode, NestedInnerQueryParseSupport nestedHelper) throws IOException { if (SCORE_FIELD_NAME.equals(fieldName)) { if (reverse) { sortFields.add(SORT_SCORE_REVERSE); @@ -198,28 +204,19 @@ private void addSortField(SearchContext context, List sortFields, Str sortFields.add(SORT_DOC); } } else { - MappedFieldType fieldType = context.smartNameFieldType(fieldName); + MappedFieldType fieldType = context.fieldMapper(fieldName); if (fieldType == null) { if (unmappedType != null) { - fieldType = context.mapperService().unmappedFieldType(unmappedType); + fieldType = context.getMapperService().unmappedFieldType(unmappedType); } else { - throw new SearchParseException(context, "No mapping found for [" + fieldName + "] in order to sort on", null); + throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on"); } } if (!fieldType.isSortable()) { - throw new SearchParseException(context, "Sorting not supported for field[" + fieldName + "]", null); + throw new QueryShardException(context, "Sorting not supported for field[" + fieldName + "]"); } - // Enable when we also know how to detect fields that do tokenize, but only emit one token - /*if (fieldMapper instanceof StringFieldMapper) { - StringFieldMapper stringFieldMapper = (StringFieldMapper) fieldMapper; - if (stringFieldMapper.fieldType().tokenized()) { - // Fail early - throw new SearchParseException(context, "Can't sort on tokenized string field[" + fieldName + "]"); - } - }*/ - // We only support AVG and SUM on number based fields if (fieldType.isNumeric() == false && (sortMode == MultiValueMode.SUM || sortMode == MultiValueMode.AVG)) { sortMode = null; @@ -230,7 +227,7 @@ private void addSortField(SearchContext context, List sortFields, Str final Nested nested; if (nestedHelper != null && nestedHelper.getPath() != null) { - BitSetProducer rootDocumentsFilter = context.bitsetFilterCache().getBitSetProducer(Queries.newNonNestedFilter()); + BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter()); Query innerDocumentsQuery; if (nestedHelper.filterFound()) { innerDocumentsQuery = nestedHelper.getInnerFilter(); @@ -242,7 +239,7 @@ private void addSortField(SearchContext context, List sortFields, Str nested = null; } - IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.fieldData().getForField(fieldType) + IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.getForField(fieldType) .comparatorSource(missing, sortMode, nested); sortFields.add(new SortField(fieldType.name(), fieldComparatorSource, reverse)); } diff --git a/core/src/main/java/org/elasticsearch/search/sort/SortParser.java b/core/src/main/java/org/elasticsearch/search/sort/SortParser.java index 727e576a85e46..519d9adb957b1 100644 --- a/core/src/main/java/org/elasticsearch/search/sort/SortParser.java +++ b/core/src/main/java/org/elasticsearch/search/sort/SortParser.java @@ -23,6 +23,8 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryShardContext; +import java.io.IOException; + /** * */ @@ -30,5 +32,5 @@ public interface SortParser { String[] names(); - SortField parse(XContentParser parser, QueryShardContext context) throws Exception; + SortField parse(XContentParser parser, QueryShardContext context) throws IOException; } diff --git a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index f7f9edbc0b2e2..84b23fb144989 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/core/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -19,6 +19,8 @@ package org.elasticsearch.search.sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.Accountable; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -30,27 +32,75 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.cache.bitset.BitsetFilterCache; +import org.elasticsearch.index.fielddata.IndexFieldDataService; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper.BuilderContext; +import org.elasticsearch.index.mapper.core.DoubleFieldMapper.DoubleFieldType; +import org.elasticsearch.index.mapper.object.ObjectMapper; +import org.elasticsearch.index.mapper.object.ObjectMapper.Nested; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.fielddata.cache.IndicesFieldDataCache; import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.script.CompiledScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptContextRegistry; +import org.elasticsearch.script.ScriptEngineRegistry; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptServiceTests.TestEngineService; +import org.elasticsearch.script.ScriptSettings; import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.IndexSettingsModule; +import org.elasticsearch.watcher.ResourceWatcherService; import org.junit.AfterClass; import org.junit.BeforeClass; import java.io.IOException; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Map; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.not; -public abstract class AbstractSortTestCase> extends ESTestCase { +public abstract class AbstractSortTestCase & SortBuilderParser> extends ESTestCase { protected static NamedWriteableRegistry namedWriteableRegistry; private static final int NUMBER_OF_TESTBUILDERS = 20; static IndicesQueriesRegistry indicesQueriesRegistry; + private static SortParseElement parseElement = new SortParseElement(); + private static ScriptService scriptService; @BeforeClass - public static void init() { + public static void init() throws IOException { + Path genericConfigFolder = createTempDir(); + Settings baseSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString()) + .put(Environment.PATH_CONF_SETTING.getKey(), genericConfigFolder) + .build(); + Environment environment = new Environment(baseSettings); + ScriptContextRegistry scriptContextRegistry = new ScriptContextRegistry(Collections.emptyList()); + ScriptEngineRegistry scriptEngineRegistry = new ScriptEngineRegistry(Collections.singletonList(new ScriptEngineRegistry + .ScriptEngineRegistration(TestEngineService.class, TestEngineService.TYPES))); + ScriptSettings scriptSettings = new ScriptSettings(scriptEngineRegistry, scriptContextRegistry); + scriptService = new ScriptService(baseSettings, environment, Collections.singleton(new TestEngineService()), + new ResourceWatcherService(baseSettings, null), scriptEngineRegistry, scriptContextRegistry, scriptSettings) { + @Override + public CompiledScript compile(Script script, ScriptContext scriptContext, Map params) { + return new CompiledScript(ScriptType.INLINE, "mockName", "test", script); + } + }; + namedWriteableRegistry = new NamedWriteableRegistry(); namedWriteableRegistry.registerPrototype(SortBuilder.class, GeoDistanceSortBuilder.PROTOTYPE); namedWriteableRegistry.registerPrototype(SortBuilder.class, ScoreSortBuilder.PROTOTYPE); @@ -97,13 +147,40 @@ public void testFromXContent() throws IOException { QueryParseContext context = new QueryParseContext(indicesQueriesRegistry); context.reset(itemParser); - SortBuilder parsedItem = testItem.fromXContent(context, elementName); + T parsedItem = testItem.fromXContent(context, elementName); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } } + /** + * test that build() outputs a {@link SortField} that is similar to the one + * we would get when parsing the xContent the sort builder is rendering out + */ + public void testBuildSortField() throws IOException { + QueryShardContext mockShardContext = createMockShardContext(); + for (int runs = 0; runs < NUMBER_OF_TESTBUILDERS; runs++) { + T sortBuilder = createTestItem(); + SortField sortField = sortBuilder.build(mockShardContext); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + builder.startObject(); + sortBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + XContentParser parser = XContentHelper.createParser(builder.bytes()); + parser.nextToken(); + List sortFields = parseElement.parse(parser, mockShardContext); + assertEquals(1, sortFields.size()); + SortField sortFieldOldStyle = sortFields.get(0); + assertEquals(sortFieldOldStyle.getField(), sortField.getField()); + assertEquals(sortFieldOldStyle.getReverse(), sortField.getReverse()); + assertEquals(sortFieldOldStyle.getType(), sortField.getType()); + } + } + /** * Test serialization and deserialization of the test sort. */ @@ -148,8 +225,50 @@ public void testEqualsAndHashcode() throws IOException { } } + private QueryShardContext createMockShardContext() { + Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_"); + IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY); + IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null); + IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), + cache, null, null); + BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() { + + @Override + public void onRemoval(ShardId shardId, Accountable accountable) { + } + + @Override + public void onCache(ShardId shardId, Accountable accountable) { + } + }); + return new QueryShardContext(idxSettings, bitsetFilterCache, ifds, null, null, scriptService, + indicesQueriesRegistry, null) { + @Override + public MappedFieldType fieldMapper(String name) { + return provideMappedFieldType(name); + } + + @Override + public ObjectMapper getObjectMapper(String name) { + BuilderContext context = new BuilderContext(Settings.EMPTY, new ContentPath()); + return new ObjectMapper.Builder<>(name).nested(Nested.newNested(false, false)).build(context); + } + }; + } + + /** + * Return a field type. We use {@link DoubleFieldType} by default since it is compatible with all sort modes + * Tests that require other field type than double can override this. + */ + protected MappedFieldType provideMappedFieldType(String name) { + DoubleFieldType doubleFieldType = new DoubleFieldType(); + doubleFieldType.setName(name); + doubleFieldType.setHasDocValues(true); + return doubleFieldType; + } + @SuppressWarnings("unchecked") - protected T copyItem(T original) throws IOException { + private T copyItem(T original) throws IOException { try (BytesStreamOutput output = new BytesStreamOutput()) { original.writeTo(output); try (StreamInput in = new NamedWriteableAwareStreamInput(StreamInput.wrap(output.bytes()), namedWriteableRegistry)) { diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 025f793016537..d00b60e0c8331 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -25,7 +25,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase> nodePlugins() { return pluginList(InternalSettingsPlugin.class); @@ -69,7 +71,7 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce */ Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point")); + assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point")); XContentBuilder d1Builder = jsonBuilder(); GeoPoint[] d1Points = {new GeoPoint(3, 2), new GeoPoint(4, 1)}; createShuffeldJSONArray(d1Builder, d1Points); @@ -95,7 +97,7 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) - .addSort(new GeoDistanceSortBuilder("location", q).sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) + .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) .execute().actionGet(); assertOrderedSearchHits(searchResponse, "d1", "d2"); assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 3, 2, DistanceUnit.KILOMETERS), 0.01d)); @@ -103,7 +105,7 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) - .addSort(new GeoDistanceSortBuilder("location", q).sortMode(SortMode.MIN).order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) + .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MIN).order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) .execute().actionGet(); assertOrderedSearchHits(searchResponse, "d2", "d1"); assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 5, 1, DistanceUnit.KILOMETERS), 0.01d)); @@ -111,7 +113,7 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) - .addSort(new GeoDistanceSortBuilder("location", q).sortMode(SortMode.MAX).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) + .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) .execute().actionGet(); assertOrderedSearchHits(searchResponse, "d1", "d2"); assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS), 0.01d)); @@ -119,18 +121,61 @@ public void testManyToManyGeoPoints() throws ExecutionException, InterruptedExce searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) - .addSort(new GeoDistanceSortBuilder("location", q).sortMode(SortMode.MAX).order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) + .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MAX).order(SortOrder.DESC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) .execute().actionGet(); assertOrderedSearchHits(searchResponse, "d2", "d1"); assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 1, 6, 2, DistanceUnit.KILOMETERS), 0.01d)); assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(2, 2, 4, 1, DistanceUnit.KILOMETERS), 0.01d)); } + public void testSingeToManyAvgMedian() throws ExecutionException, InterruptedException, IOException { + /** + * q = (0, 0) + * + * d1 = (0, 1), (0, 4), (0, 10); so avg. distance is 5, median distance is 4 + * d2 = (0, 1), (0, 5), (0, 6); so avg. distance is 4, median distance is 5 + */ + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); + Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); + assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point")); + XContentBuilder d1Builder = jsonBuilder(); + GeoPoint[] d1Points = {new GeoPoint(0, 1), new GeoPoint(0, 4), new GeoPoint(0, 10)}; + createShuffeldJSONArray(d1Builder, d1Points); + + XContentBuilder d2Builder = jsonBuilder(); + GeoPoint[] d2Points = {new GeoPoint(0, 1), new GeoPoint(0, 5), new GeoPoint(0, 6)}; + createShuffeldJSONArray(d2Builder, d2Points); + + logger.info("d1: {}", d1Builder); + logger.info("d2: {}", d2Builder); + indexRandom(true, + client().prepareIndex("index", "type", "d1").setSource(d1Builder), + client().prepareIndex("index", "type", "d2").setSource(d2Builder)); + ensureYellow(); + GeoPoint q = new GeoPoint(0,0); + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.AVG).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) + .execute().actionGet(); + assertOrderedSearchHits(searchResponse, "d2", "d1"); + assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 4, DistanceUnit.KILOMETERS), 0.01d)); + assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 5, DistanceUnit.KILOMETERS), 0.01d)); + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .addSort(new GeoDistanceSortBuilder(LOCATION_FIELD, q).sortMode(SortMode.MEDIAN).order(SortOrder.ASC).geoDistance(GeoDistance.PLANE).unit(DistanceUnit.KILOMETERS)) + .execute().actionGet(); + assertOrderedSearchHits(searchResponse, "d1", "d2"); + assertThat((Double)searchResponse.getHits().getAt(0).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 4, DistanceUnit.KILOMETERS), 0.01d)); + assertThat((Double)searchResponse.getHits().getAt(1).getSortValues()[0], closeTo(GeoDistance.PLANE.calculate(0, 0, 0, 5, DistanceUnit.KILOMETERS), 0.01d)); + } + protected void createShuffeldJSONArray(XContentBuilder builder, GeoPoint[] pointsArray) throws IOException { List points = new ArrayList<>(); points.addAll(Arrays.asList(pointsArray)); builder.startObject(); - builder.startArray("location"); + builder.startArray(LOCATION_FIELD); int numPoints = points.size(); for (int i = 0; i < numPoints; i++) { builder.value(points.remove(randomInt(points.size() - 1))); @@ -154,7 +199,7 @@ public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionExcept */ Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.CURRENT); Settings settings = Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", "location", "type=geo_point")); + assertAcked(prepareCreate("index").setSettings(settings).addMapping("type", LOCATION_FIELD, "type=geo_point")); XContentBuilder d1Builder = jsonBuilder(); GeoPoint[] d1Points = {new GeoPoint(2.5, 1), new GeoPoint(2.75, 2), new GeoPoint(3, 3), new GeoPoint(3.25, 4)}; createShuffeldJSONArray(d1Builder, d1Points); @@ -177,13 +222,13 @@ public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionExcept int at = randomInt(3 - i); if (randomBoolean()) { if (geoDistanceSortBuilder == null) { - geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", qHashes.get(at)); + geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, qHashes.get(at)); } else { geoDistanceSortBuilder.geohashes(qHashes.get(at)); } } else { if (geoDistanceSortBuilder == null) { - geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", qPoints.get(at)); + geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, qPoints.get(at)); } else { geoDistanceSortBuilder.points(qPoints.get(at)); } @@ -211,15 +256,15 @@ public void testManyToManyGeoPointsWithDifferentFormats() throws ExecutionExcept } public void testSinglePointGeoDistanceSort() throws ExecutionException, InterruptedException, IOException { - assertAcked(prepareCreate("index").addMapping("type", "location", "type=geo_point")); + assertAcked(prepareCreate("index").addMapping("type", LOCATION_FIELD, "type=geo_point")); indexRandom(true, - client().prepareIndex("index", "type", "d1").setSource(jsonBuilder().startObject().startObject("location").field("lat", 1).field("lon", 1).endObject().endObject()), - client().prepareIndex("index", "type", "d2").setSource(jsonBuilder().startObject().startObject("location").field("lat", 1).field("lon", 2).endObject().endObject())); + client().prepareIndex("index", "type", "d1").setSource(jsonBuilder().startObject().startObject(LOCATION_FIELD).field("lat", 1).field("lon", 1).endObject().endObject()), + client().prepareIndex("index", "type", "d2").setSource(jsonBuilder().startObject().startObject(LOCATION_FIELD).field("lat", 1).field("lon", 2).endObject().endObject())); ensureYellow(); String hashPoint = "s037ms06g7h0"; - GeoDistanceSortBuilder geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", hashPoint); + GeoDistanceSortBuilder geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, hashPoint); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -227,7 +272,7 @@ public void testSinglePointGeoDistanceSort() throws ExecutionException, Interrup .execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); - geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", new GeoPoint(2, 2)); + geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, new GeoPoint(2, 2)); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -235,7 +280,7 @@ public void testSinglePointGeoDistanceSort() throws ExecutionException, Interrup .execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); - geoDistanceSortBuilder = new GeoDistanceSortBuilder("location", 2, 2); + geoDistanceSortBuilder = new GeoDistanceSortBuilder(LOCATION_FIELD, 2, 2); searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -246,28 +291,28 @@ public void testSinglePointGeoDistanceSort() throws ExecutionException, Interrup searchResponse = client() .prepareSearch() .setSource( - new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location", 2.0, 2.0) + new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0) .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); searchResponse = client() .prepareSearch() .setSource( - new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location", "s037ms06g7h0") + new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, "s037ms06g7h0") .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); searchResponse = client() .prepareSearch() .setSource( - new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location", 2.0, 2.0) + new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0) .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE))).execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); searchResponse = client() .prepareSearch() .setSource( - new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort("location", 2.0, 2.0) + new SearchSourceBuilder().sort(SortBuilders.geoDistanceSort(LOCATION_FIELD, 2.0, 2.0) .unit(DistanceUnit.KILOMETERS).geoDistance(GeoDistance.PLANE) .ignoreMalformed(true).coerce(true))).execute().actionGet(); checkCorrectSortOrderForGeoSort(searchResponse); diff --git a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java index 50e4aeeb71b73..1fc0a8cacde7d 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/GeoDistanceSortBuilderTests.java @@ -26,6 +26,8 @@ import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import org.elasticsearch.index.query.QueryParseContext; import org.elasticsearch.test.geo.RandomGeoGenerator; @@ -89,6 +91,13 @@ protected GeoDistanceSortBuilder createTestItem() { return result; } + @Override + protected MappedFieldType provideMappedFieldType(String name) { + MappedFieldType clone = GeoPointFieldMapper.Defaults.FIELD_TYPE.clone(); + clone.setName(name); + return clone; + } + private static SortMode mode(SortMode original) { SortMode result; do { @@ -167,7 +176,6 @@ protected GeoDistanceSortBuilder mutate(GeoDistanceSortBuilder original) throws break; } return result; - } public void testSortModeSumIsRejectedInSetter() { diff --git a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java index 405c1c43e775d..efac1bab19b76 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java +++ b/core/src/test/java/org/elasticsearch/search/sort/RandomSortDataGenerator.java @@ -26,6 +26,9 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.test.ESTestCase; +import java.util.HashSet; +import java.util.Set; + public class RandomSortDataGenerator { private RandomSortDataGenerator() { // this is a helper class only, doesn't need a constructor @@ -44,7 +47,7 @@ public static QueryBuilder nestedFilter(QueryBuilder original) { break; default: case 2: - nested = new TermQueryBuilder(ESTestCase.randomAsciiOfLengthBetween(1, 10), ESTestCase.randomAsciiOfLengthBetween(1, 10)); + nested = new TermQueryBuilder(ESTestCase.randomAsciiOfLengthBetween(1, 10), ESTestCase.randomDouble()); break; } nested.boost((float) ESTestCase.randomDoubleBetween(0, 10, false)); @@ -61,8 +64,14 @@ public static String randomAscii(String original) { } public static SortMode mode(SortMode original) { + Set set = new HashSet<>(); + set.add(original); + return mode(set); + } + + public static SortMode mode(Set except) { SortMode mode = ESTestCase.randomFrom(SortMode.values()); - while (mode.equals(original)) { + while (except.contains(mode)) { mode = ESTestCase.randomFrom(SortMode.values()); } return mode; diff --git a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java index 091a6c3002a26..b28285e096c96 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/ScriptSortBuilderTests.java @@ -33,18 +33,29 @@ import org.junit.rules.ExpectedException; import java.io.IOException; +import java.util.HashSet; +import java.util.Set; public class ScriptSortBuilderTests extends AbstractSortTestCase { @Override protected ScriptSortBuilder createTestItem() { + ScriptSortType type = randomBoolean() ? ScriptSortType.NUMBER : ScriptSortType.STRING; ScriptSortBuilder builder = new ScriptSortBuilder(new Script(randomAsciiOfLengthBetween(5, 10)), - randomBoolean() ? ScriptSortType.NUMBER : ScriptSortType.STRING); + type); if (randomBoolean()) { - builder.order(RandomSortDataGenerator.order(builder.order())); + builder.order(RandomSortDataGenerator.order(builder.order())); } if (randomBoolean()) { - builder.sortMode(RandomSortDataGenerator.mode(builder.sortMode())); + if (type == ScriptSortType.NUMBER) { + builder.sortMode(RandomSortDataGenerator.mode(builder.sortMode())); + } else { + Set exceptThis = new HashSet<>(); + exceptThis.add(SortMode.SUM); + exceptThis.add(SortMode.AVG); + exceptThis.add(SortMode.MEDIAN); + builder.sortMode(RandomSortDataGenerator.mode(exceptThis)); + } } if (randomBoolean()) { builder.setNestedFilter(RandomSortDataGenerator.nestedFilter(builder.getNestedFilter())); @@ -68,7 +79,7 @@ protected ScriptSortBuilder mutate(ScriptSortBuilder original) throws IOExceptio result = new ScriptSortBuilder(script, type.equals(ScriptSortType.NUMBER) ? ScriptSortType.STRING : ScriptSortType.NUMBER); } result.order(original.order()); - if (original.sortMode() != null) { + if (original.sortMode() != null && result.type() == ScriptSortType.NUMBER) { result.sortMode(original.sortMode()); } result.setNestedFilter(original.getNestedFilter()); @@ -85,7 +96,16 @@ protected ScriptSortBuilder mutate(ScriptSortBuilder original) throws IOExceptio } break; case 1: - result.sortMode(RandomSortDataGenerator.mode(original.sortMode())); + if (original.type() == ScriptSortType.NUMBER) { + result.sortMode(RandomSortDataGenerator.mode(original.sortMode())); + } else { + // script sort type String only allows MIN and MAX, so we only switch + if (original.sortMode() == SortMode.MIN) { + result.sortMode(SortMode.MAX); + } else { + result.sortMode(SortMode.MIN); + } + } break; case 2: result.setNestedFilter(RandomSortDataGenerator.nestedFilter(original.getNestedFilter())); @@ -238,4 +258,14 @@ public void testParseUnexpectedToken() throws IOException { exceptionRule.expectMessage("unexpected token [START_ARRAY]"); ScriptSortBuilder.PROTOTYPE.fromXContent(context, null); } + + /** + * script sort of type {@link ScriptSortType} does not work with {@link SortMode#AVG}, {@link SortMode#MEDIAN} or {@link SortMode#SUM} + */ + public void testBadSortMode() throws IOException { + ScriptSortBuilder builder = new ScriptSortBuilder(new Script("something"), ScriptSortType.STRING); + exceptionRule.expect(IllegalArgumentException.class); + exceptionRule.expectMessage("script sort of type [string] doesn't support mode"); + builder.sortMode(SortMode.fromString(randomFrom(new String[]{"avg", "median", "sum"}))); + } }