Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add build() method to SortBuilder implementations #17146

Merged
merged 5 commits into from
Mar 22, 2016
Merged

Add build() method to SortBuilder implementations #17146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1908d6d
Add build() method to SortBuilder implementations
cbuescher Mar 15, 2016
d9de129
Adressing review comments
cbuescher Mar 17, 2016
4c3d889
Adding tests for median sort mode to GeoDistanceSortBuilderIT
cbuescher Mar 18, 2016
25da6b2
Merge branch 'master' into sort-add-build
cbuescher Mar 22, 2016
697174d
Make sure to use nestedScope levels when building nested filters
cbuescher Mar 22, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions core/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ public QueryShardContext(IndexSettings indexSettings, BitsetFilterCache bitsetFi
this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields();
this.indicesQueriesRegistry = indicesQueriesRegistry;
this.percolatorQueryCache = percolatorQueryCache;
this.nestedScope = new NestedScope();
}

public QueryShardContext(QueryShardContext source) {
Expand All @@ -113,6 +114,7 @@ public QueryShardContext(QueryShardContext source) {
}


@Override
public QueryShardContext clone() {
return new QueryShardContext(indexSettings, bitsetFilterCache, indexFieldDataService, mapperService, similarityService, scriptService, indicesQueriesRegistry, percolatorQueryCache);
}
Expand Down
61 changes: 59 additions & 2 deletions core/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.search.sort;

import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
Expand All @@ -27,8 +28,14 @@
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.query.QueryShardException;
import org.elasticsearch.search.MultiValueMode;

import java.io.IOException;
import java.util.Objects;
Expand All @@ -47,6 +54,13 @@ public class FieldSortBuilder extends SortBuilder<FieldSortBuilder> implements S
public static final ParseField SORT_MODE = new ParseField("mode");
public static final ParseField UNMAPPED_TYPE = new ParseField("unmapped_type");

/**
* special field name to sort by index order
*/
public static final String DOC_FIELD_NAME = "_doc";
private static final SortField SORT_DOC = new SortField(null, SortField.Type.DOC);
private static final SortField SORT_DOC_REVERSE = new SortField(null, SortField.Type.DOC, true);

private final String fieldName;

private Object missing;
Expand Down Expand Up @@ -161,7 +175,7 @@ public SortMode sortMode() {
* TODO should the above getters and setters be deprecated/ changed in
* favour of real getters and setters?
*/
public FieldSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
public FieldSortBuilder setNestedFilter(QueryBuilder<?> nestedFilter) {
this.nestedFilter = nestedFilter;
return this;
}
Expand All @@ -170,7 +184,7 @@ public FieldSortBuilder setNestedFilter(QueryBuilder nestedFilter) {
* Returns the nested filter that the nested objects should match with in
* order to be taken into account for sorting.
*/
public QueryBuilder getNestedFilter() {
public QueryBuilder<?> getNestedFilter() {
return this.nestedFilter;
}

Expand Down Expand Up @@ -219,6 +233,49 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
return builder;
}

@Override
public SortField build(QueryShardContext context) throws IOException {
if (DOC_FIELD_NAME.equals(fieldName)) {
if (order == SortOrder.DESC) {
return SORT_DOC_REVERSE;
} else {
return SORT_DOC;
}
} else {
MappedFieldType fieldType = context.fieldMapper(fieldName);
if (fieldType == null) {
if (unmappedType != null) {
fieldType = context.getMapperService().unmappedFieldType(unmappedType);
} else {
throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on");
}
}

if (!fieldType.isSortable()) {
throw new QueryShardException(context, "Sorting not supported for field[" + fieldName + "]");
}

MultiValueMode localSortMode = null;
if (sortMode != null) {
localSortMode = MultiValueMode.fromString(sortMode.toString());
}

if (fieldType.isNumeric() == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
}

boolean reverse = (order == SortOrder.DESC);
if (localSortMode == null) {
localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

By directly using order here we could delete the variable "reverse" above.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmpf - should first read through and comment afterwards. I saw only now that we need "reverse" further below anyway :/

}

final Nested nested = resolveNested(context, nestedPath, nestedFilter);
IndexFieldData.XFieldComparatorSource fieldComparatorSource = context.getForField(fieldType)
.comparatorSource(missing, localSortMode, nested);
return new SortField(fieldType.name(), fieldComparatorSource, reverse);
}
}

@Override
public boolean equals(Object other) {
if (this == other) {
Expand Down
143 changes: 123 additions & 20 deletions core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,36 @@

package org.elasticsearch.search.sort;

import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.BitSet;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.geo.GeoDistance.FixedSourceDistance;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
import org.elasticsearch.index.fielddata.IndexGeoPointFieldData;
import org.elasticsearch.index.fielddata.MultiGeoPointValues;
import org.elasticsearch.index.fielddata.NumericDoubleValues;
import org.elasticsearch.index.fielddata.SortedNumericDoubleValues;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.search.MultiValueMode;

import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -45,6 +64,14 @@ public class GeoDistanceSortBuilder extends SortBuilder<GeoDistanceSortBuilder>
public static final String NAME = "_geo_distance";
public static final boolean DEFAULT_COERCE = false;
public static final boolean DEFAULT_IGNORE_MALFORMED = false;
public static final ParseField UNIT_FIELD = new ParseField("unit");
public static final ParseField REVERSE_FIELD = new ParseField("reverse");
public static final ParseField DISTANCE_TYPE_FIELD = new ParseField("distance_type");
public static final ParseField COERCE_FIELD = new ParseField("coerce", "normalize");
public static final ParseField IGNORE_MALFORMED_FIELD = new ParseField("ignore_malformed");
public static final ParseField SORTMODE_FIELD = new ParseField("mode", "sort_mode");
public static final ParseField NESTED_PATH_FIELD = new ParseField("nested_path");
public static final ParseField NESTED_FILTER_FIELD = new ParseField("nested_filter");

static final GeoDistanceSortBuilder PROTOTYPE = new GeoDistanceSortBuilder("", -1, -1);

Expand Down Expand Up @@ -280,22 +307,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
}
builder.endArray();

builder.field("unit", unit);
builder.field("distance_type", geoDistance.name().toLowerCase(Locale.ROOT));
builder.field(UNIT_FIELD.getPreferredName(), unit);
builder.field(DISTANCE_TYPE_FIELD.getPreferredName(), geoDistance.name().toLowerCase(Locale.ROOT));
builder.field(ORDER_FIELD.getPreferredName(), order);

if (sortMode != null) {
builder.field("mode", sortMode);
builder.field(SORTMODE_FIELD.getPreferredName(), sortMode);
}

if (nestedPath != null) {
builder.field("nested_path", nestedPath);
builder.field(NESTED_PATH_FIELD.getPreferredName(), nestedPath);
}
if (nestedFilter != null) {
builder.field("nested_filter", nestedFilter, params);
builder.field(NESTED_FILTER_FIELD.getPreferredName(), nestedFilter, params);
}
builder.field("coerce", coerce);
builder.field("ignore_malformed", ignoreMalformed);
builder.field(COERCE_FIELD.getPreferredName(), coerce);
builder.field(IGNORE_MALFORMED_FIELD.getPreferredName(), ignoreMalformed);

builder.endObject();
return builder;
Expand Down Expand Up @@ -383,6 +410,7 @@ public GeoDistanceSortBuilder readFrom(StreamInput in) throws IOException {
@Override
public GeoDistanceSortBuilder fromXContent(QueryParseContext context, String elementName) throws IOException {
XContentParser parser = context.parser();
ParseFieldMatcher parseFieldMatcher = context.parseFieldMatcher();
String fieldName = null;
List<GeoPoint> geoPoints = new ArrayList<>();
DistanceUnit unit = DistanceUnit.DEFAULT;
Expand All @@ -405,40 +433,37 @@ public GeoDistanceSortBuilder fromXContent(QueryParseContext context, String ele

fieldName = currentName;
} else if (token == XContentParser.Token.START_OBJECT) {
// the json in the format of -> field : { lat : 30, lon : 12 }
if ("nested_filter".equals(currentName) || "nestedFilter".equals(currentName)) {
// TODO Note to remember: while this is kept as a QueryBuilder internally,
// we need to make sure to call toFilter() on it once on the shard
// (e.g. in the new build() method)
if (parseFieldMatcher.match(currentName, NESTED_FILTER_FIELD)) {
nestedFilter = context.parseInnerQueryBuilder();
} else {
// the json in the format of -> field : { lat : 30, lon : 12 }
fieldName = currentName;
GeoPoint point = new GeoPoint();
GeoUtils.parseGeoPoint(parser, point);
geoPoints.add(point);
}
} else if (token.isValue()) {
if ("reverse".equals(currentName)) {
if (parseFieldMatcher.match(currentName, REVERSE_FIELD)) {
order = parser.booleanValue() ? SortOrder.DESC : SortOrder.ASC;
} else if ("order".equals(currentName)) {
} else if (parseFieldMatcher.match(currentName, ORDER_FIELD)) {
order = SortOrder.fromString(parser.text());
} else if ("unit".equals(currentName)) {
} else if (parseFieldMatcher.match(currentName, UNIT_FIELD)) {
unit = DistanceUnit.fromString(parser.text());
} else if ("distance_type".equals(currentName) || "distanceType".equals(currentName)) {
} else if (parseFieldMatcher.match(currentName, DISTANCE_TYPE_FIELD)) {
geoDistance = GeoDistance.fromString(parser.text());
} else if ("coerce".equals(currentName) || "normalize".equals(currentName)) {
} else if (parseFieldMatcher.match(currentName, COERCE_FIELD)) {
coerce = parser.booleanValue();
if (coerce == true) {
ignoreMalformed = true;
}
} else if ("ignore_malformed".equals(currentName)) {
} else if (parseFieldMatcher.match(currentName, IGNORE_MALFORMED_FIELD)) {
boolean ignore_malformed_value = parser.booleanValue();
if (coerce == false) {
ignoreMalformed = ignore_malformed_value;
}
} else if ("sort_mode".equals(currentName) || "sortMode".equals(currentName) || "mode".equals(currentName)) {
} else if (parseFieldMatcher.match(currentName, SORTMODE_FIELD)) {
sortMode = SortMode.fromString(parser.text());
} else if ("nested_path".equals(currentName) || "nestedPath".equals(currentName)) {
} else if (parseFieldMatcher.match(currentName, NESTED_PATH_FIELD)) {
nestedPath = parser.text();
} else {
GeoPoint point = new GeoPoint();
Expand All @@ -461,7 +486,85 @@ public GeoDistanceSortBuilder fromXContent(QueryParseContext context, String ele
result.coerce(coerce);
result.ignoreMalformed(ignoreMalformed);
return result;
}

@Override
public SortField build(QueryShardContext context) throws IOException {
final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0);
// validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed on 2.x created indexes
List<GeoPoint> localPoints = new ArrayList<GeoPoint>();
for (GeoPoint geoPoint : this.points) {
localPoints.add(new GeoPoint(geoPoint));
}

if (!indexCreatedBeforeV2_0 && !ignoreMalformed) {
for (GeoPoint point : localPoints) {
if (GeoUtils.isValidLatitude(point.lat()) == false) {
throw new ElasticsearchParseException("illegal latitude value [{}] for [GeoDistanceSort]", point.lat());
}
if (GeoUtils.isValidLongitude(point.lon()) == false) {
throw new ElasticsearchParseException("illegal longitude value [{}] for [GeoDistanceSort]", point.lon());
}
}
}

if (coerce) {
for (GeoPoint point : localPoints) {
GeoUtils.normalizePoint(point, coerce, coerce);
}
}

boolean reverse = (order == SortOrder.DESC);
final MultiValueMode finalSortMode;
if (sortMode == null) {
finalSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
} else {
finalSortMode = MultiValueMode.fromString(sortMode.toString());
}

MappedFieldType fieldType = context.fieldMapper(fieldName);
if (fieldType == null) {
throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort");
}
final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType);
final FixedSourceDistance[] distances = new FixedSourceDistance[localPoints.size()];
for (int i = 0; i< localPoints.size(); i++) {
distances[i] = geoDistance.fixedSourceDistance(localPoints.get(i).lat(), localPoints.get(i).lon(), unit);
}

final Nested nested = resolveNested(context, nestedPath, nestedFilter);

IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {

@Override
public SortField.Type reducedType() {
return SortField.Type.DOUBLE;
}

@Override
public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException {
return new FieldComparator.DoubleComparator(numHits, null, null) {
@Override
protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
final SortedNumericDoubleValues distanceValues = GeoDistance.distanceValues(geoPointValues, distances);
final NumericDoubleValues selectedValues;
if (nested == null) {
selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE);
} else {
final BitSet rootDocs = nested.rootDocs(context);
final DocIdSetIterator innerDocs = nested.innerDocs(context);
selectedValues = finalSortMode.select(distanceValues, Double.MAX_VALUE, rootDocs, innerDocs,
context.reader().maxDoc());
}
return selectedValues.getRawDoubleValues();
}
};
}

};

return new SortField(fieldName, geoDistanceComparatorSource, reverse);
}

static void parseGeoPoints(XContentParser parser, List<GeoPoint> geoPoints) throws IOException {
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public String[] names() {
}

@Override
public SortField parse(XContentParser parser, QueryShardContext context) throws Exception {
public SortField parse(XContentParser parser, QueryShardContext context) throws IOException {
String fieldName = null;
List<GeoPoint> geoPoints = new ArrayList<>();
DistanceUnit unit = DistanceUnit.DEFAULT;
Expand Down
12 changes: 12 additions & 0 deletions core/src/main/java/org/elasticsearch/search/sort/ScoreSortBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.search.sort;

import org.apache.lucene.search.SortField;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParsingException;
Expand All @@ -27,6 +28,7 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.index.query.QueryShardContext;

import java.io.IOException;
import java.util.Objects;
Expand All @@ -40,6 +42,8 @@ public class ScoreSortBuilder extends SortBuilder<ScoreSortBuilder> implements S
static final ScoreSortBuilder PROTOTYPE = new ScoreSortBuilder();
public static final ParseField REVERSE_FIELD = new ParseField("reverse");
public static final ParseField ORDER_FIELD = new ParseField("order");
private static final SortField SORT_SCORE = new SortField(null, SortField.Type.SCORE);
private static final SortField SORT_SCORE_REVERSE = new SortField(null, SortField.Type.SCORE, true);

public ScoreSortBuilder() {
// order defaults to desc when sorting on the _score
Expand Down Expand Up @@ -84,6 +88,14 @@ public ScoreSortBuilder fromXContent(QueryParseContext context, String elementNa
return result;
}

public SortField build(QueryShardContext context) {
if (order == SortOrder.DESC) {
return SORT_SCORE;
} else {
return SORT_SCORE_REVERSE;
}
}

@Override
public boolean equals(Object object) {
if (this == object) {
Expand Down
Loading