diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 274970e0668a0..7e2c0241d7d16 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -114,6 +114,11 @@ and reject the whole document. |`coerce` |If `true` unclosed linear rings in polygons will be automatically closed. | `false` +|`doc_values` |Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. +| `true` for BKD-backed geo_shape, `false` for prefix tree indexing strategy + |======================================================================= diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java index f67c404dc4188..ffd55a45fe5f9 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java @@ -53,6 +53,20 @@ public class Geohash { /** Bit encoded representation of the latitude of north pole */ private static final long MAX_LAT_BITS = (0x1L << (PRECISION * 5 / 2)) - 1; + // Below code is adapted from the spatial4j library (GeohashUtils.java) Apache 2.0 Licensed + private static final double[] precisionToLatHeight, precisionToLonWidth; + static { + precisionToLatHeight = new double[PRECISION + 1]; + precisionToLonWidth = new double[PRECISION + 1]; + precisionToLatHeight[0] = 90*2; + precisionToLonWidth[0] = 180*2; + boolean even = false; + for(int i = 1; i <= PRECISION; i++) { + precisionToLatHeight[i] = precisionToLatHeight[i-1] / (even ? 8 : 4); + precisionToLonWidth[i] = precisionToLonWidth[i-1] / (even ? 4 : 8); + even = ! even; + } + } // no instance: private Geohash() { @@ -97,6 +111,16 @@ public static Rectangle toBoundingBox(final String geohash) { } } + /** Array of geohashes 1 level below the baseGeohash. Sorted. */ + public static String[] getSubGeohashes(String baseGeohash) { + String[] hashes = new String[BASE_32.length]; + for (int i = 0; i < BASE_32.length; i++) {//note: already sorted + char c = BASE_32[i]; + hashes[i] = baseGeohash+c; + } + return hashes; + } + /** * Calculate all neighbors of a given geohash cell. * @@ -201,6 +225,13 @@ public static final String getNeighbor(String geohash, int level, int dx, int dy } } + /** + * Encode a string geohash to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) + */ + public static final long longEncode(String hash) { + return longEncode(hash, hash.length()); + } + /** * Encode lon/lat to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) */ @@ -297,7 +328,6 @@ private static long encodeLatLon(final double lat, final double lon) { return BitUtil.interleave(latEnc, lonEnc) >>> 2; } - /** encode latitude to integer */ public static int encodeLatitude(double latitude) { // the maximum possible value cannot be encoded without overflow @@ -316,6 +346,16 @@ public static int encodeLongitude(double longitude) { return (int) Math.floor(longitude / LON_DECODE); } + /** approximate width of geohash tile for a specific precision in degrees */ + public static double lonWidthInDegrees(int precision) { + return precisionToLonWidth[precision]; + } + + /** approximate height of geohash tile for a specific precision in degrees */ + public static double latHeightInDegrees(int precision) { + return precisionToLatHeight[precision]; + } + /** returns the latitude value from the string based geohash */ public static final double decodeLatitude(final String geohash) { return decodeLatitude(Geohash.mortonEncode(geohash)); diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java index 0b16aaf9dcde0..5eaf416e92d63 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java @@ -27,9 +27,9 @@ import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; /** * ValueSource to return non-zero if a field is missing. @@ -44,8 +44,8 @@ final class GeoEmptyValueSource extends ValueSource { @Override @SuppressWarnings("rawtypes") // ValueSource uses a rawtype public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException { - AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf); - final MultiGeoPointValues values = leafData.getGeoPointValues(); + AtomicGeoFieldData leafData = (AtomicGeoFieldData) fieldData.load(leaf); + final MultiGeoValues values = leafData.getGeoValues(); return new DoubleDocValues(this) { @Override public double doubleVal(int doc) throws IOException { diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java index fd812dac5a3a8..1722820f944e4 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java @@ -27,9 +27,9 @@ import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; /** * ValueSource to return latitudes as a double "stream" for geopoint fields @@ -44,13 +44,13 @@ final class GeoLatitudeValueSource extends ValueSource { @Override @SuppressWarnings("rawtypes") // ValueSource uses a rawtype public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException { - AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf); - final MultiGeoPointValues values = leafData.getGeoPointValues(); + AtomicGeoFieldData leafData = (AtomicGeoFieldData) fieldData.load(leaf); + final MultiGeoValues values = leafData.getGeoValues(); return new DoubleDocValues(this) { @Override public double doubleVal(int doc) throws IOException { if (values.advanceExact(doc)) { - return values.nextValue().getLat(); + return values.nextValue().lat(); } else { return 0.0; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java index fd05d92d62350..9c27d90426c2c 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java @@ -27,9 +27,9 @@ import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; /** * ValueSource to return longitudes as a double "stream" for geopoint fields @@ -44,13 +44,13 @@ final class GeoLongitudeValueSource extends ValueSource { @Override @SuppressWarnings("rawtypes") // ValueSource uses a rawtype public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException { - AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf); - final MultiGeoPointValues values = leafData.getGeoPointValues(); + AtomicGeoFieldData leafData = (AtomicGeoFieldData) fieldData.load(leaf); + final MultiGeoValues values = leafData.getGeoValues(); return new DoubleDocValues(this) { @Override public double doubleVal(int doc) throws IOException { if (values.advanceExact(doc)) { - return values.nextValue().getLon(); + return values.nextValue().lon(); } else { return 0.0; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java new file mode 100644 index 0000000000000..5eb84f64aa6bd --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java @@ -0,0 +1,316 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.metrics.CompensatedSum; + +import static org.elasticsearch.common.geo.DimensionalShapeType.LINE; +import static org.elasticsearch.common.geo.DimensionalShapeType.POINT; +import static org.elasticsearch.common.geo.DimensionalShapeType.POLYGON; + +/** + * This class keeps a running Kahan-sum of coordinates + * that are to be averaged in {@link TriangleTreeWriter} for use + * as the centroid of a shape. + */ +public class CentroidCalculator { + CompensatedSum compSumX; + CompensatedSum compSumY; + CompensatedSum compSumWeight; + private CentroidCalculatorVisitor visitor; + private DimensionalShapeType dimensionalShapeType; + + public CentroidCalculator(Geometry geometry) { + this.compSumX = new CompensatedSum(0, 0); + this.compSumY = new CompensatedSum(0, 0); + this.compSumWeight = new CompensatedSum(0, 0); + this.dimensionalShapeType = null; + this.visitor = new CentroidCalculatorVisitor(this); + geometry.visit(visitor); + this.dimensionalShapeType = visitor.calculator.dimensionalShapeType; + } + + /** + * adds a single coordinate to the running sum and count of coordinates + * for centroid calculation + * @param x the x-coordinate of the point + * @param y the y-coordinate of the point + * @param weight the associated weight of the coordinate + */ + private void addCoordinate(double x, double y, double weight, DimensionalShapeType dimensionalShapeType) { + // x and y can be infinite due to really small areas and rounding problems + if (Double.isFinite(x) && Double.isFinite(y)) { + if (this.dimensionalShapeType == null || this.dimensionalShapeType == dimensionalShapeType) { + compSumX.add(x * weight); + compSumY.add(y * weight); + compSumWeight.add(weight); + this.dimensionalShapeType = dimensionalShapeType; + } else if (dimensionalShapeType.compareTo(this.dimensionalShapeType) > 0) { + // reset counters + compSumX.reset(x * weight, 0); + compSumY.reset(y * weight, 0); + compSumWeight.reset(weight, 0); + this.dimensionalShapeType = dimensionalShapeType; + } + } + } + + /** + * Adjusts the existing calculator to add the running sum and count + * from another {@link CentroidCalculator}. This is used to keep + * a running count of points from different sub-shapes of a single + * geo-shape field + * + * @param otherCalculator the other centroid calculator to add from + */ + public void addFrom(CentroidCalculator otherCalculator) { + int compared = dimensionalShapeType.compareTo(otherCalculator.dimensionalShapeType); + if (compared < 0) { + dimensionalShapeType = otherCalculator.dimensionalShapeType; + this.compSumX = otherCalculator.compSumX; + this.compSumY = otherCalculator.compSumY; + this.compSumWeight = otherCalculator.compSumWeight; + + } else if (compared == 0) { + this.compSumX.add(otherCalculator.compSumX.value()); + this.compSumY.add(otherCalculator.compSumY.value()); + this.compSumWeight.add(otherCalculator.compSumWeight.value()); + } // else (compared > 0) do not modify centroid calculation since otherCalculator is of lower dimension than this calculator + } + + /** + * @return the x-coordinate centroid + */ + public double getX() { + // normalization required due to floating point precision errors + return GeoUtils.normalizeLon(compSumX.value() / compSumWeight.value()); + } + + /** + * @return the y-coordinate centroid + */ + public double getY() { + // normalization required due to floating point precision errors + return GeoUtils.normalizeLat(compSumY.value() / compSumWeight.value()); + } + + /** + * @return the sum of all the weighted coordinates summed in the calculator + */ + public double sumWeight() { + return compSumWeight.value(); + } + + /** + * @return the highest dimensional shape type summed in the calculator + */ + public DimensionalShapeType getDimensionalShapeType() { + return dimensionalShapeType; + } + + private static class CentroidCalculatorVisitor implements GeometryVisitor { + + private final CentroidCalculator calculator; + + private CentroidCalculatorVisitor(CentroidCalculator calculator) { + this.calculator = calculator; + } + + @Override + public Void visit(Circle circle) { + throw new IllegalArgumentException("invalid shape type found [Circle] while calculating centroid"); + } + + @Override + public Void visit(GeometryCollection collection) { + for (Geometry shape : collection) { + shape.visit(this); + } + return null; + } + + @Override + public Void visit(Line line) { + if (calculator.dimensionalShapeType != POLYGON) { + visitLine(line.length(), line::getX, line::getY); + } + return null; + } + + @Override + public Void visit(LinearRing ring) { + throw new IllegalArgumentException("invalid shape type found [LinearRing] while calculating centroid"); + } + + + @Override + public Void visit(MultiLine multiLine) { + if (calculator.getDimensionalShapeType() != POLYGON) { + for (Line line : multiLine) { + visit(line); + } + } + return null; + } + + @Override + public Void visit(MultiPoint multiPoint) { + if (calculator.getDimensionalShapeType() == null || calculator.getDimensionalShapeType() == POINT) { + for (Point point : multiPoint) { + visit(point); + } + } + return null; + } + + @Override + public Void visit(MultiPolygon multiPolygon) { + for (Polygon polygon : multiPolygon) { + visit(polygon); + } + return null; + } + + @Override + public Void visit(Point point) { + if (calculator.getDimensionalShapeType() == null || calculator.getDimensionalShapeType() == POINT) { + visitPoint(point.getX(), point.getY()); + } + return null; + } + + @Override + public Void visit(Polygon polygon) { + // check area of polygon + + double[] centroidX = new double[1 + polygon.getNumberOfHoles()]; + double[] centroidY = new double[1 + polygon.getNumberOfHoles()]; + double[] weight = new double[1 + polygon.getNumberOfHoles()]; + visitLinearRing(polygon.getPolygon().length(), polygon.getPolygon()::getX, polygon.getPolygon()::getY, false, + centroidX, centroidY, weight, 0); + for (int i = 0; i < polygon.getNumberOfHoles(); i++) { + visitLinearRing(polygon.getHole(i).length(), polygon.getHole(i)::getX, polygon.getHole(i)::getY, true, + centroidX, centroidY, weight, i + 1); + } + + double sumWeight = 0; + for (double w : weight) { + sumWeight += w; + } + + if (sumWeight == 0 && calculator.dimensionalShapeType != POLYGON) { + visitLine(polygon.getPolygon().length(), polygon.getPolygon()::getX, polygon.getPolygon()::getY); + } else { + for (int i = 0; i < 1 + polygon.getNumberOfHoles(); i++) { + calculator.addCoordinate(centroidX[i], centroidY[i], weight[i], POLYGON); + } + } + + return null; + } + + @Override + public Void visit(Rectangle rectangle) { + double sumX = rectangle.getMaxX() + rectangle.getMinX(); + double sumY = rectangle.getMaxY() + rectangle.getMinY(); + double diffX = rectangle.getMaxX() - rectangle.getMinX(); + double diffY = rectangle.getMaxY() - rectangle.getMinY(); + if (diffX != 0 && diffY != 0) { + calculator.addCoordinate(sumX / 2, sumY / 2, Math.abs(diffX * diffY), POLYGON); + } else if (diffX != 0) { + calculator.addCoordinate(sumX / 2, rectangle.getMinY(), diffX, LINE); + } else if (diffY != 0) { + calculator.addCoordinate(rectangle.getMinX(), sumY / 2, diffY, LINE); + } else { + visitPoint(rectangle.getMinX(), rectangle.getMinY()); + } + return null; + } + + + private void visitPoint(double x, double y) { + calculator.addCoordinate(x, y, 1.0, POINT); + } + + private void visitLine(int length, CoordinateSupplier x, CoordinateSupplier y) { + // check line has length + double originDiffX = x.get(0) - x.get(1); + double originDiffY = y.get(0) - y.get(1); + if (originDiffX != 0 || originDiffY != 0) { + // a line's centroid is calculated by summing the center of each + // line segment weighted by the line segment's length in degrees + for (int i = 0; i < length - 1; i++) { + double diffX = x.get(i) - x.get(i + 1); + double diffY = y.get(i) - y.get(i + 1); + double xAvg = (x.get(i) + x.get(i + 1)) / 2; + double yAvg = (y.get(i) + y.get(i + 1)) / 2; + double weight = Math.sqrt(diffX * diffX + diffY * diffY); + calculator.addCoordinate(xAvg, yAvg, weight, LINE); + } + } else { + visitPoint(x.get(0), y.get(0)); + } + } + + private void visitLinearRing(int length, CoordinateSupplier x, CoordinateSupplier y, boolean isHole, + double[] centroidX, double[] centroidY, double[] weight, int idx) { + // implementation of calculation defined in + // https://www.seas.upenn.edu/~sys502/extra_materials/Polygon%20Area%20and%20Centroid.pdf + // + // centroid of a ring is a weighted coordinate based on the ring's area. + // the sign of the area is positive for the outer-shell of a polygon and negative for the holes + + int sign = isHole ? -1 : 1; + double totalRingArea = 0.0; + for (int i = 0; i < length - 1; i++) { + totalRingArea += (x.get(i) * y.get(i + 1)) - (x.get(i + 1) * y.get(i)); + } + totalRingArea = totalRingArea / 2; + + double sumX = 0.0; + double sumY = 0.0; + for (int i = 0; i < length - 1; i++) { + double twiceArea = (x.get(i) * y.get(i + 1)) - (x.get(i + 1) * y.get(i)); + sumX += twiceArea * (x.get(i) + x.get(i + 1)); + sumY += twiceArea * (y.get(i) + y.get(i + 1)); + } + centroidX[idx] = sumX / (6 * totalRingArea); + centroidY[idx] = sumY / (6 * totalRingArea); + weight[idx] = sign * Math.abs(totalRingArea); + } + } + + @FunctionalInterface + private interface CoordinateSupplier { + double get(int idx); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/CoordinateEncoder.java b/server/src/main/java/org/elasticsearch/common/geo/CoordinateEncoder.java new file mode 100644 index 0000000000000..c1be8545f87d8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/CoordinateEncoder.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +/** + * Interface for classes that help encode double-valued spatial coordinates x/y to + * their integer-encoded serialized form and decode them back + */ +public interface CoordinateEncoder { + int encodeX(double x); + int encodeY(double y); + double decodeX(int x); + double decodeY(int y); +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java b/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java new file mode 100644 index 0000000000000..ce9ce5a3a9d79 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java @@ -0,0 +1,50 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.ShapeType; + +/** + * Like {@link ShapeType} but has specific + * types for when the geometry is a {@link GeometryCollection} and + * more information about what the highest-dimensional sub-shape + * is. + */ +public enum DimensionalShapeType { + POINT, + LINE, + POLYGON; + + private static DimensionalShapeType[] values = values(); + + public static DimensionalShapeType fromOrdinalByte(byte ordinal) { + return values[Byte.toUnsignedInt(ordinal)]; + } + + public void writeTo(ByteBuffersDataOutput out) { + out.writeByte((byte) ordinal()); + } + + public static DimensionalShapeType readFrom(ByteArrayDataInput in) { + return fromOrdinalByte(in.readByte()); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java new file mode 100644 index 0000000000000..67bb96600fd82 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -0,0 +1,295 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteBuffersDataOutput; + +import java.io.IOException; +import java.util.Objects; + +/** + * Object representing the extent of a geometry object within a {@link TriangleTreeWriter}. + */ +public class Extent { + + public int top; + public int bottom; + public int negLeft; + public int negRight; + public int posLeft; + public int posRight; + + private static final byte NONE_SET = 0; + private static final byte POSITIVE_SET = 1; + private static final byte NEGATIVE_SET = 2; + private static final byte CROSSES_LAT_AXIS = 3; + private static final byte ALL_SET = 4; + + + public Extent() { + this.top = Integer.MIN_VALUE; + this.bottom = Integer.MAX_VALUE; + this.negLeft = Integer.MAX_VALUE; + this.negRight = Integer.MIN_VALUE; + this.posLeft = Integer.MAX_VALUE; + this.posRight = Integer.MIN_VALUE; + } + + public Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + this.top = top; + this.bottom = bottom; + this.negLeft = negLeft; + this.negRight = negRight; + this.posLeft = posLeft; + this.posRight = posRight; + } + + public void reset(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + this.top = top; + this.bottom = bottom; + this.negLeft = negLeft; + this.negRight = negRight; + this.posLeft = posLeft; + this.posRight = posRight; + } + + /** + * Adds the extent of two points representing a bounding box's bottom-left + * and top-right points. The bounding box must not cross the dateline. + * + * @param bottomLeftX the bottom-left x-coordinate + * @param bottomLeftY the bottom-left y-coordinate + * @param topRightX the top-right x-coordinate + * @param topRightY the top-right y-coordinate + */ + public void addRectangle(int bottomLeftX, int bottomLeftY, int topRightX, int topRightY) { + assert bottomLeftX <= topRightX; + assert bottomLeftY <= topRightY; + this.bottom = Math.min(this.bottom, bottomLeftY); + this.top = Math.max(this.top, topRightY); + if (bottomLeftX < 0 && topRightX < 0) { + this.negLeft = Math.min(this.negLeft, bottomLeftX); + this.negRight = Math.max(this.negRight, topRightX); + } else if (bottomLeftX < 0) { + this.negLeft = Math.min(this.negLeft, bottomLeftX); + this.posRight = Math.max(this.posRight, topRightX); + // this signal the extent cannot be wrapped around the dateline + this.negRight = 0; + this.posLeft = 0; + } else { + this.posLeft = Math.min(this.posLeft, bottomLeftX); + this.posRight = Math.max(this.posRight, topRightX); + } + } + + static void readFromCompressed(ByteArrayDataInput input, Extent extent) { + final int top = input.readInt(); + final int bottom = Math.toIntExact(top - input.readVLong()); + final int negLeft; + final int negRight; + final int posLeft; + final int posRight; + byte type = input.readByte(); + switch (type) { + case NONE_SET: + negLeft = Integer.MAX_VALUE; + negRight = Integer.MIN_VALUE; + posLeft = Integer.MAX_VALUE; + posRight = Integer.MIN_VALUE; + break; + case POSITIVE_SET: + posLeft = input.readVInt(); + posRight = Math.toIntExact(input.readVLong() + posLeft); + negLeft = Integer.MAX_VALUE; + negRight = Integer.MIN_VALUE; + break; + case NEGATIVE_SET: + negRight = -input.readVInt(); + negLeft = Math.toIntExact(negRight - input.readVLong()); + posLeft = Integer.MAX_VALUE; + posRight = Integer.MIN_VALUE; + break; + case CROSSES_LAT_AXIS: + posRight = input.readVInt(); + negLeft = -input.readVInt(); + posLeft = 0; + negRight = 0; + break; + case ALL_SET: + posLeft = input.readVInt(); + posRight = Math.toIntExact(input.readVLong() + posLeft); + negRight = -input.readVInt(); + negLeft = Math.toIntExact(negRight - input.readVLong()); + break; + default: + throw new IllegalArgumentException("invalid extent values-set byte read [" + type + "]"); + } + extent.reset(top, bottom, negLeft, negRight, posLeft, posRight); + } + + void writeCompressed(ByteBuffersDataOutput output) throws IOException { + output.writeInt(this.top); + output.writeVLong((long) this.top - this.bottom); + byte type; + if (this.negLeft == Integer.MAX_VALUE && this.negRight == Integer.MIN_VALUE) { + if (this.posLeft == Integer.MAX_VALUE && this.posRight == Integer.MIN_VALUE) { + type = NONE_SET; + } else { + type = POSITIVE_SET; + } + } else if (this.posLeft == Integer.MAX_VALUE && this.posRight == Integer.MIN_VALUE) { + type = NEGATIVE_SET; + } else { + if (posLeft == 0 && negRight == 0) { + type = CROSSES_LAT_AXIS; + } else { + type = ALL_SET; + } + } + output.writeByte(type); + switch (type) { + case NONE_SET : break; + case POSITIVE_SET: + output.writeVInt(this.posLeft); + output.writeVLong((long) this.posRight - this.posLeft); + break; + case NEGATIVE_SET: + output.writeVInt(-this.negRight); + output.writeVLong((long) this.negRight - this.negLeft); + break; + case CROSSES_LAT_AXIS: + output.writeVInt(this.posRight); + output.writeVInt(-this.negLeft); + break; + case ALL_SET: + output.writeVInt(this.posLeft); + output.writeVLong((long) this.posRight - this.posLeft); + output.writeVInt(-this.negRight); + output.writeVLong((long) this.negRight - this.negLeft); + break; + default: + throw new IllegalArgumentException("invalid extent values-set byte read [" + type + "]"); + } + } + + /** + * calculates the extent of a point, which is the point itself. + * @param x the x-coordinate of the point + * @param y the y-coordinate of the point + * @return the extent of the point + */ + public static Extent fromPoint(int x, int y) { + return new Extent(y, y, + x < 0 ? x : Integer.MAX_VALUE, + x < 0 ? x : Integer.MIN_VALUE, + x >= 0 ? x : Integer.MAX_VALUE, + x >= 0 ? x : Integer.MIN_VALUE); + } + + /** + * calculates the extent of two points representing a bounding box's bottom-left + * and top-right points. It is important that these points accurately represent the + * bottom-left and top-right of the extent since there is no validation being done. + * + * @param bottomLeftX the bottom-left x-coordinate + * @param bottomLeftY the bottom-left y-coordinate + * @param topRightX the top-right x-coordinate + * @param topRightY the top-right y-coordinate + * @return the extent of the two points + */ + static Extent fromPoints(int bottomLeftX, int bottomLeftY, int topRightX, int topRightY) { + int negLeft = Integer.MAX_VALUE; + int negRight = Integer.MIN_VALUE; + int posLeft = Integer.MAX_VALUE; + int posRight = Integer.MIN_VALUE; + if (bottomLeftX < 0 && topRightX < 0) { + negLeft = bottomLeftX; + negRight = topRightX; + } else if (bottomLeftX < 0) { + negLeft = bottomLeftX; + posRight = topRightX; + // this signal the extent cannot be wrapped around the dateline + negRight = 0; + posLeft = 0; + } else { + posLeft = bottomLeftX; + posRight = topRightX; + } + return new Extent(topRightY, bottomLeftY, negLeft, negRight, posLeft, posRight); + } + + /** + * @return the minimum y-coordinate of the extent + */ + public int minY() { + return bottom; + } + + /** + * @return the maximum y-coordinate of the extent + */ + public int maxY() { + return top; + } + + /** + * @return the absolute minimum x-coordinate of the extent, whether it is positive or negative. + */ + public int minX() { + return Math.min(negLeft, posLeft); + } + + /** + * @return the absolute maximum x-coordinate of the extent, whether it is positive or negative. + */ + public int maxX() { + return Math.max(negRight, posRight); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Extent extent = (Extent) o; + return top == extent.top && + bottom == extent.bottom && + negLeft == extent.negLeft && + negRight == extent.negRight && + posLeft == extent.posLeft && + posRight == extent.posRight; + } + + @Override + public int hashCode() { + return Objects.hash(top, bottom, negLeft, negRight, posLeft, posRight); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("["); + builder.append("top = " + top + ", "); + builder.append("bottom = " + bottom + ", "); + builder.append("negLeft = " + negLeft + ", "); + builder.append("negRight = " + negRight + ", "); + builder.append("posLeft = " + posLeft + ", "); + builder.append("posRight = " + posRight + "]"); + return builder.toString(); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoRelation.java b/server/src/main/java/org/elasticsearch/common/geo/GeoRelation.java new file mode 100644 index 0000000000000..ac8f17d3ca81b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoRelation.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +/** + * Enum for capturing relationships between a shape + * and a query + */ +public enum GeoRelation { + QUERY_CROSSES, + QUERY_INSIDE, + QUERY_DISJOINT +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoder.java b/server/src/main/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoder.java new file mode 100644 index 0000000000000..79666ffb1b1fa --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoder.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +import org.apache.lucene.geo.GeoEncodingUtils; + +public final class GeoShapeCoordinateEncoder implements CoordinateEncoder { + public static final GeoShapeCoordinateEncoder INSTANCE = new GeoShapeCoordinateEncoder(); + + @Override + public int encodeX(double x) { + if (x == Double.NEGATIVE_INFINITY) { + return Integer.MIN_VALUE; + } + if (x == Double.POSITIVE_INFINITY) { + return Integer.MAX_VALUE; + } + return GeoEncodingUtils.encodeLongitude(x); + } + + @Override + public int encodeY(double y) { + if (y == Double.NEGATIVE_INFINITY) { + return Integer.MIN_VALUE; + } + if (y == Double.POSITIVE_INFINITY) { + return Integer.MAX_VALUE; + } + return GeoEncodingUtils.encodeLatitude(y); + } + + @Override + public double decodeX(int x) { + return GeoEncodingUtils.decodeLongitude(x); + } + + @Override + public double decodeY(int y) { + return GeoEncodingUtils.decodeLatitude(y); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index d33f90043b1d5..ab7001c94d612 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -32,8 +32,7 @@ import org.elasticsearch.common.xcontent.support.MapXContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.GeoPointValues; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; @@ -608,10 +607,11 @@ public static double planeDistance(double lat1, double lon1, double lat2, double */ public static SortedNumericDoubleValues distanceValues(final GeoDistance distance, final DistanceUnit unit, - final MultiGeoPointValues geoPointValues, + final MultiGeoValues geoPointValues, final GeoPoint... fromPoints) { - final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues); + final MultiGeoValues singleValues = FieldData.unwrapSingleton(geoPointValues); if (singleValues != null && fromPoints.length == 1) { + assert singleValues.docValueCount() == 1; return FieldData.singleton(new NumericDoubleValues() { @Override @@ -622,7 +622,7 @@ public boolean advanceExact(int doc) throws IOException { @Override public double doubleValue() throws IOException { final GeoPoint from = fromPoints[0]; - final GeoPoint to = singleValues.geoPointValue(); + final MultiGeoValues.GeoValue to = singleValues.nextValue(); return distance.calculate(from.lat(), from.lon(), to.lat(), to.lon(), unit); } @@ -635,7 +635,7 @@ public boolean advanceExact(int target) throws IOException { resize(geoPointValues.docValueCount() * fromPoints.length); int v = 0; for (int i = 0; i < geoPointValues.docValueCount(); ++i) { - final GeoPoint point = geoPointValues.nextValue(); + final MultiGeoValues.GeoValue point = geoPointValues.nextValue(); for (GeoPoint from : fromPoints) { values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit); v++; diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java new file mode 100644 index 0000000000000..b6b06e5088162 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -0,0 +1,430 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; + +import static org.apache.lucene.geo.GeoUtils.orient; + +/** + * A tree reusable reader for a previous serialized {@link org.elasticsearch.geometry.Geometry} using + * {@link TriangleTreeWriter}. + * + * This class supports checking bounding box + * relations against the serialized triangle tree. + * + * ----------------------------------------- + * | The binary format of the tree | + * ----------------------------------------- + * ----------------------------------------- -- + * | centroid-x-coord (4 bytes) | | + * ----------------------------------------- | + * | centroid-y-coord (4 bytes) | | + * ----------------------------------------- | + * | DimensionalShapeType (1 byte) | | Centroid-related header + * ----------------------------------------- | + * | Sum of weights (VLong 1-8 bytes) | | + * ----------------------------------------- -- + * | Extent (var-encoding) | + * ----------------------------------------- + * | Triangle Tree | + * ----------------------------------------- + * ----------------------------------------- + */ +public class TriangleTreeReader { + private final ByteArrayDataInput input; + private final CoordinateEncoder coordinateEncoder; + private final Tile2D tile2D; + private final Extent extent; + private int treeOffset; + private int docValueOffset; + + public TriangleTreeReader(CoordinateEncoder coordinateEncoder) { + this.coordinateEncoder = coordinateEncoder; + this.tile2D = new Tile2D(); + this.extent = new Extent(); + this.input = new ByteArrayDataInput(); + } + + public void reset(BytesRef bytesRef) throws IOException { + this.input.reset(bytesRef.bytes, bytesRef.offset, bytesRef.length); + docValueOffset = bytesRef.offset; + treeOffset = 0; + } + + /** + * returns the bounding box of the geometry in the format [minX, maxX, minY, maxY]. + */ + public Extent getExtent() { + if (treeOffset == 0) { + getSumCentroidWeight(); // skip CENTROID_HEADER + var-long sum-weight + Extent.readFromCompressed(input, extent); + treeOffset = input.getPosition(); + } else { + input.setPosition(treeOffset); + } + return extent; + } + + /** + * returns the X coordinate of the centroid. + */ + public double getCentroidX() { + input.setPosition(docValueOffset + 0); + return coordinateEncoder.decodeX(input.readInt()); + } + + /** + * returns the Y coordinate of the centroid. + */ + public double getCentroidY() { + input.setPosition(docValueOffset + 4); + return coordinateEncoder.decodeY(input.readInt()); + } + + public DimensionalShapeType getDimensionalShapeType() { + input.setPosition(docValueOffset + 8); + return DimensionalShapeType.readFrom(input); + } + + public double getSumCentroidWeight() { + input.setPosition(docValueOffset + 9); + return Double.longBitsToDouble(input.readVLong()); + } + + /** + * Compute the relation with the provided bounding box. If the result is CELL_INSIDE_QUERY + * then the bounding box is within the shape. + */ + public GeoRelation relateTile(int minX, int minY, int maxX, int maxY) { + Extent extent = getExtent(); + int thisMaxX = extent.maxX(); + int thisMinX = extent.minX(); + int thisMaxY = extent.maxY(); + int thisMinY = extent.minY(); + + // exclude north and east boundary intersections with tiles from intersection consideration + // for consistent tiling definition of shapes on the boundaries of tiles + if ((thisMinX >= maxX || thisMaxX < minX || thisMinY > maxY || thisMaxY <= minY)) { + // shapes are disjoint + return GeoRelation.QUERY_DISJOINT; + } + if (minX <= thisMinX && maxX >= thisMaxX && minY <= thisMinY && maxY >= thisMaxY) { + // the rectangle fully contains the shape + return GeoRelation.QUERY_CROSSES; + } + // quick checks failed, need to traverse the tree + GeoRelation rel = GeoRelation.QUERY_DISJOINT; + tile2D.setValues(minX, maxX, minY, maxY); + byte metadata = input.readByte(); + if ((metadata & 1 << 2) == 1 << 2) { // component in this node is a point + int x = Math.toIntExact(thisMaxX - input.readVLong()); + int y = Math.toIntExact(thisMaxY - input.readVLong()); + if (tile2D.contains(x, y)) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = x; + } else if ((metadata & 1 << 3) == 1 << 3) { // component in this node is a line + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + if (tile2D.intersectsLine(aX, aY, bX, bY)) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + } else { // component in this node is a triangle + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + int cX = Math.toIntExact(thisMaxX - input.readVLong()); + int cY = Math.toIntExact(thisMaxY - input.readVLong()); + boolean ab = (metadata & 1 << 4) == 1 << 4; + boolean bc = (metadata & 1 << 5) == 1 << 5; + boolean ca = (metadata & 1 << 6) == 1 << 6; + rel = tile2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); + if (rel == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + } + if ((metadata & 1 << 0) == 1 << 0) { // left != null + GeoRelation left = relateTile(tile2D, false, thisMaxX, thisMaxY); + if (left == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (left == GeoRelation.QUERY_INSIDE) { + rel = left; + } + } + if ((metadata & 1 << 1) == 1 << 1) { // right != null + if (tile2D.maxX >= thisMinX) { + GeoRelation right = relateTile(tile2D, false, thisMaxX, thisMaxY); + if (right == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (right == GeoRelation.QUERY_INSIDE) { + rel = right; + } + } + } + + return rel; + } + + private GeoRelation relateTile(Tile2D tile2D, boolean splitX, int parentMaxX, int parentMaxY) { + int thisMaxX = Math.toIntExact(parentMaxX - input.readVLong()); + int thisMaxY = Math.toIntExact(parentMaxY - input.readVLong()); + GeoRelation rel = GeoRelation.QUERY_DISJOINT; + int size = input.readVInt(); + if (tile2D.minY <= thisMaxY && tile2D.minX <= thisMaxX) { + byte metadata = input.readByte(); + int thisMinX; + int thisMinY; + if ((metadata & 1 << 2) == 1 << 2) { // component in this node is a point + int x = Math.toIntExact(thisMaxX - input.readVLong()); + int y = Math.toIntExact(thisMaxY - input.readVLong()); + if (tile2D.contains(x, y)) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = x; + thisMinY = y; + } else if ((metadata & 1 << 3) == 1 << 3) { // component in this node is a line + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + if (tile2D.intersectsLine(aX, aY, bX, bY)) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + thisMinY = Math.min(aY, bY); + } else { // component in this node is a triangle + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + int cX = Math.toIntExact(thisMaxX - input.readVLong()); + int cY = Math.toIntExact(thisMaxY - input.readVLong()); + boolean ab = (metadata & 1 << 4) == 1 << 4; + boolean bc = (metadata & 1 << 5) == 1 << 5; + boolean ca = (metadata & 1 << 6) == 1 << 6; + rel = tile2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); + if (rel == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + thisMinY = Math.min(Math.min(aY, bY), cY); + } + if ((metadata & 1 << 0) == 1 << 0) { // left != null + GeoRelation left = relateTile(tile2D, !splitX, thisMaxX, thisMaxY); + if (left == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (left == GeoRelation.QUERY_INSIDE) { + rel = left; + } + } + if ((metadata & 1 << 1) == 1 << 1) { // right != null + int rightSize = input.readVInt(); + if ((splitX == false && tile2D.maxY >= thisMinY) || (splitX && tile2D.maxX >= thisMinX)) { + GeoRelation right = relateTile(tile2D, !splitX, thisMaxX, thisMaxY); + if (right == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (right == GeoRelation.QUERY_INSIDE) { + rel = right; + } + } else { + input.skipBytes(rightSize); + } + } + } else { + input.skipBytes(size); + } + return rel; + } + + private static class Tile2D { + + protected int minX; + protected int maxX; + protected int minY; + protected int maxY; + + Tile2D() { + } + + private void setValues(int minX, int maxX, int minY, int maxY) { + this.minX = minX; + this.maxX = maxX; + this.minY = minY; + this.maxY = maxY; + } + + /** + * Checks if the rectangle contains the provided point + **/ + public boolean contains(int x, int y) { + return (x <= minX || x > maxX || y < minY || y >= maxY) == false; + } + + /** + * Checks if the rectangle intersects the provided triangle + **/ + private boolean intersectsLine(int aX, int aY, int bX, int bY) { + // 1. query contains any triangle points + if (contains(aX, aY) || contains(bX, bY)) { + return true; + } + + // compute bounding box of triangle + int tMinX = StrictMath.min(aX, bX); + int tMaxX = StrictMath.max(aX, bX); + int tMinY = StrictMath.min(aY, bY); + int tMaxY = StrictMath.max(aY, bY); + + // 2. check bounding boxes are disjoint + if (tMaxX <= minX || tMinX > maxX || tMinY > maxY || tMaxY <= minY) { + return false; + } + + // 4. last ditch effort: check crossings + if (edgeIntersectsQuery(aX, aY, bX, bY)) { + return true; + } + return false; + } + + /** + * Checks if the rectangle intersects the provided triangle + **/ + private GeoRelation relateTriangle(int aX, int aY, boolean ab, int bX, int bY, boolean bc, int cX, int cY, boolean ca) { + // compute bounding box of triangle + int tMinX = StrictMath.min(StrictMath.min(aX, bX), cX); + int tMaxX = StrictMath.max(StrictMath.max(aX, bX), cX); + int tMinY = StrictMath.min(StrictMath.min(aY, bY), cY); + int tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY); + + // 1. check bounding boxes are disjoint, where north and east boundaries are not considered as crossing + if (tMaxX <= minX || tMinX > maxX || tMinY > maxY || tMaxY <= minY) { + return GeoRelation.QUERY_DISJOINT; + } + + // 2. query contains any triangle points + if (contains(aX, aY) || contains(bX, bY) || contains(cX, cY)) { + return GeoRelation.QUERY_CROSSES; + } + + boolean within = false; + if (edgeIntersectsQuery(aX, aY, bX, bY)) { + if (ab) { + return GeoRelation.QUERY_CROSSES; + } + within = true; + } + + // right + if (edgeIntersectsQuery(bX, bY, cX, cY)) { + if (bc) { + return GeoRelation.QUERY_CROSSES; + } + within = true; + } + + if (edgeIntersectsQuery(cX, cY, aX, aY)) { + if (ca) { + return GeoRelation.QUERY_CROSSES; + } + within = true; + } + + if (within || pointInTriangle(tMinX, tMaxX, tMinY, tMaxY, minX, minY, aX, aY, bX, bY, cX, cY)) { + return GeoRelation.QUERY_INSIDE; + } + + return GeoRelation.QUERY_DISJOINT; + } + + /** + * returns true if the edge (defined by (ax, ay) (bx, by)) intersects the query + */ + private boolean edgeIntersectsQuery(int ax, int ay, int bx, int by) { + // shortcut: check bboxes of edges are disjoint + if (boxesAreDisjoint(Math.min(ax, bx), Math.max(ax, bx), Math.min(ay, by), Math.max(ay, by), + minX, maxX, minY, maxY)) { + return false; + } + + // top + if (orient(ax, ay, bx, by, minX, maxY) * orient(ax, ay, bx, by, maxX, maxY) <= 0 && + orient(minX, maxY, maxX, maxY, ax, ay) * orient(minX, maxY, maxX, maxY, bx, by) <= 0) { + return true; + } + + // right + if (orient(ax, ay, bx, by, maxX, maxY) * orient(ax, ay, bx, by, maxX, minY) <= 0 && + orient(maxX, maxY, maxX, minY, ax, ay) * orient(maxX, maxY, maxX, minY, bx, by) <= 0) { + return true; + } + + // bottom + if (orient(ax, ay, bx, by, maxX, minY) * orient(ax, ay, bx, by, minX, minY) <= 0 && + orient(maxX, minY, minX, minY, ax, ay) * orient(maxX, minY, minX, minY, bx, by) <= 0) { + return true; + } + + // left + if (orient(ax, ay, bx, by, minX, minY) * orient(ax, ay, bx, by, minX, maxY) <= 0 && + orient(minX, minY, minX, maxY, ax, ay) * orient(minX, minY, minX, maxY, bx, by) <= 0) { + return true; + } + + return false; + } + + /** + * Compute whether the given x, y point is in a triangle; uses the winding order method + */ + private static boolean pointInTriangle(double minX, double maxX, double minY, double maxY, double x, double y, + double aX, double aY, double bX, double bY, double cX, double cY) { + //check the bounding box because if the triangle is degenerated, e.g points and lines, we need to filter out + //coplanar points that are not part of the triangle. + if (x >= minX && x <= maxX && y >= minY && y <= maxY) { + int a = orient(x, y, aX, aY, bX, bY); + int b = orient(x, y, bX, bY, cX, cY); + if (a == 0 || b == 0 || a < 0 == b < 0) { + int c = orient(x, y, cX, cY, aX, aY); + return c == 0 || (c < 0 == (b < 0 || a < 0)); + } + return false; + } else { + return false; + } + } + + /** + * utility method to check if two boxes are disjoint + */ + private static boolean boxesAreDisjoint(final int aMinX, final int aMaxX, final int aMinY, final int aMaxY, + final int bMinX, final int bMaxX, final int bMinY, final int bMaxY) { + return (aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java new file mode 100644 index 0000000000000..e9ecb844b875b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java @@ -0,0 +1,266 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.util.ArrayUtil; + +import java.io.IOException; +import java.util.Comparator; +import java.util.List; + +/** + * This is a tree-writer that serializes a list of {@link ShapeField.DecodedTriangle} as an interval tree + * into a byte array. + */ +public class TriangleTreeWriter { + + private final TriangleTreeNode node; + private final CoordinateEncoder coordinateEncoder; + private final CentroidCalculator centroidCalculator; + private Extent extent; + + public TriangleTreeWriter(List triangles, CoordinateEncoder coordinateEncoder, + CentroidCalculator centroidCalculator) { + this.coordinateEncoder = coordinateEncoder; + this.centroidCalculator = centroidCalculator; + this.extent = new Extent(); + this.node = build(triangles); + } + + /*** Serialize the interval tree in the provided data output */ + public void writeTo(ByteBuffersDataOutput out) throws IOException { + out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); + out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); + centroidCalculator.getDimensionalShapeType().writeTo(out); + out.writeVLong(Double.doubleToLongBits(centroidCalculator.sumWeight())); + extent.writeCompressed(out); + node.writeTo(out); + } + + private void addToExtent(TriangleTreeNode treeNode) { + extent.addRectangle(treeNode.minX, treeNode.minY, treeNode.maxX, treeNode.maxY); + } + + private TriangleTreeNode build(List triangles) { + if (triangles.size() == 1) { + TriangleTreeNode triangleTreeNode = new TriangleTreeNode(triangles.get(0)); + addToExtent(triangleTreeNode); + return triangleTreeNode; + } + TriangleTreeNode[] nodes = new TriangleTreeNode[triangles.size()]; + for (int i = 0; i < triangles.size(); i++) { + nodes[i] = new TriangleTreeNode(triangles.get(i)); + addToExtent(nodes[i]); + } + return createTree(nodes, 0, triangles.size() - 1, true); + } + + /** Creates tree from sorted components (with range low and high inclusive) */ + private TriangleTreeNode createTree(TriangleTreeNode[] components, int low, int high, boolean splitX) { + if (low > high) { + return null; + } + final int mid = (low + high) >>> 1; + if (low < high) { + Comparator comparator; + if (splitX) { + comparator = Comparator.comparingInt((TriangleTreeNode left) -> left.minX).thenComparingInt(left -> left.maxX); + } else { + comparator = Comparator.comparingInt((TriangleTreeNode left) -> left.minY).thenComparingInt(left -> left.maxY); + } + ArrayUtil.select(components, low, high + 1, mid, comparator); + } + TriangleTreeNode newNode = components[mid]; + // find children + newNode.left = createTree(components, low, mid - 1, !splitX); + newNode.right = createTree(components, mid + 1, high, !splitX); + + // pull up max values to this node + if (newNode.left != null) { + newNode.maxX = Math.max(newNode.maxX, newNode.left.maxX); + newNode.maxY = Math.max(newNode.maxY, newNode.left.maxY); + } + if (newNode.right != null) { + newNode.maxX = Math.max(newNode.maxX, newNode.right.maxX); + newNode.maxY = Math.max(newNode.maxY, newNode.right.maxY); + } + return newNode; + } + + /** Represents an inner node of the tree. */ + private static class TriangleTreeNode { + /** type of component */ + public enum TYPE { + POINT, LINE, TRIANGLE + } + /** minimum latitude of this geometry's bounding box area */ + private int minY; + /** maximum latitude of this geometry's bounding box area */ + private int maxY; + /** minimum longitude of this geometry's bounding box area */ + private int minX; + /** maximum longitude of this geometry's bounding box area */ + private int maxX; + // child components, or null. + private TriangleTreeNode left; + private TriangleTreeNode right; + /** root node of edge tree */ + private final ShapeField.DecodedTriangle component; + /** component type */ + private final TYPE type; + + private TriangleTreeNode(ShapeField.DecodedTriangle component) { + this.minY = Math.min(Math.min(component.aY, component.bY), component.cY); + this.maxY = Math.max(Math.max(component.aY, component.bY), component.cY); + this.minX = Math.min(Math.min(component.aX, component.bX), component.cX); + this.maxX = Math.max(Math.max(component.aX, component.bX), component.cX); + this.component = component; + this.type = getType(component); + } + + private static TYPE getType(ShapeField.DecodedTriangle triangle) { + // the issue in lucene: https://github.com/apache/lucene-solr/pull/927 + // can help here + if (triangle.aX == triangle.bX && triangle.aY == triangle.bY) { + if (triangle.aX == triangle.cX && triangle.aY == triangle.cY) { + return TYPE.POINT; + } + return TYPE.LINE; + } else if ((triangle.aX == triangle.cX && triangle.aY == triangle.cY) || + (triangle.bX == triangle.cX && triangle.bY == triangle.cY)) { + return TYPE.LINE; + } else { + return TYPE.TRIANGLE; + } + } + + private void writeTo(ByteBuffersDataOutput out) throws IOException { + ByteBuffersDataOutput scratchBuffer = ByteBuffersDataOutput.newResettableInstance(); + writeMetadata(out); + writeComponent(out); + if (left != null) { + left.writeNode(out, maxX, maxY, scratchBuffer); + } + if (right != null) { + right.writeNode(out, maxX, maxY, scratchBuffer); + } + } + + private void writeNode(ByteBuffersDataOutput out, int parentMaxX, int parentMaxY, + ByteBuffersDataOutput scratchBuffer) throws IOException { + out.writeVLong((long) parentMaxX - maxX); + out.writeVLong((long) parentMaxY - maxY); + int size = nodeSize(false, parentMaxX, parentMaxY, scratchBuffer); + out.writeVInt(size); + writeMetadata(out); + writeComponent(out); + if (left != null) { + left.writeNode(out, maxX, maxY, scratchBuffer); + } + if (right != null) { + int rightSize = right.nodeSize(true, maxX, maxY, scratchBuffer); + out.writeVInt(rightSize); + right.writeNode(out, maxX, maxY, scratchBuffer); + } + } + + private void writeMetadata(ByteBuffersDataOutput out) { + byte metadata = 0; + metadata |= (left != null) ? (1 << 0) : 0; + metadata |= (right != null) ? (1 << 1) : 0; + if (type == TYPE.POINT) { + metadata |= (1 << 2); + } else if (type == TYPE.LINE) { + metadata |= (1 << 3); + } else { + metadata |= (component.ab) ? (1 << 4) : 0; + metadata |= (component.bc) ? (1 << 5) : 0; + metadata |= (component.ca) ? (1 << 6) : 0; + } + out.writeByte(metadata); + } + + private void writeComponent(ByteBuffersDataOutput out) throws IOException { + if (type == TYPE.POINT) { + out.writeVLong((long) maxX - component.aX); + out.writeVLong((long) maxY - component.aY); + } else if (type == TYPE.LINE) { + out.writeVLong((long) maxX - component.aX); + out.writeVLong((long) maxY - component.aY); + out.writeVLong((long) maxX - component.bX); + out.writeVLong((long) maxY - component.bY); + } else { + out.writeVLong((long) maxX - component.aX); + out.writeVLong((long) maxY - component.aY); + out.writeVLong((long) maxX - component.bX); + out.writeVLong((long) maxY - component.bY); + out.writeVLong((long) maxX - component.cX); + out.writeVLong((long) maxY - component.cY); + } + } + + private int nodeSize(boolean includeBox, int parentMaxX, int parentMaxY, ByteBuffersDataOutput scratchBuffer) throws IOException { + int size =0; + size++; //metadata + size += componentSize(scratchBuffer); + if (left != null) { + size += left.nodeSize(true, maxX, maxY, scratchBuffer); + } + if (right != null) { + int rightSize = right.nodeSize(true, maxX, maxY, scratchBuffer); + scratchBuffer.reset(); + scratchBuffer.writeVLong(rightSize); + size += scratchBuffer.size(); // jump size + size += rightSize; + } + if (includeBox) { + int jumpSize = size; + scratchBuffer.reset(); + scratchBuffer.writeVLong((long) parentMaxX - maxX); + scratchBuffer.writeVLong((long) parentMaxY - maxY); + scratchBuffer.writeVLong(jumpSize); + size += scratchBuffer.size(); // box size + } + return size; + } + + private int componentSize(ByteBuffersDataOutput scratchBuffer) throws IOException { + scratchBuffer.reset(); + if (type == TYPE.POINT) { + scratchBuffer.writeVLong((long) maxX - component.aX); + scratchBuffer.writeVLong((long) maxY - component.aY); + } else if (type == TYPE.LINE) { + scratchBuffer.writeVLong((long) maxX - component.aX); + scratchBuffer.writeVLong((long) maxY - component.aY); + scratchBuffer.writeVLong((long) maxX - component.bX); + scratchBuffer.writeVLong((long) maxY - component.bY); + } else { + scratchBuffer.writeVLong((long) maxX - component.aX); + scratchBuffer.writeVLong((long) maxY - component.aY); + scratchBuffer.writeVLong((long) maxX - component.bX); + scratchBuffer.writeVLong((long) maxY - component.bY); + scratchBuffer.writeVLong((long) maxX - component.cX); + scratchBuffer.writeVLong((long) maxY - component.cY); + } + return Math.toIntExact(scratchBuffer.size()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java index 0668fcb85fef9..d245f3950433e 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java @@ -110,6 +110,14 @@ public long readLong() throws IOException { } } + public void position(int newPosition) throws IOException { + buffer.position(newPosition); + } + + public int position() throws IOException { + return buffer.position(); + } + @Override public void reset() throws IOException { buffer.reset(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoFieldData.java similarity index 81% rename from server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java rename to server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoFieldData.java index a496e1d93ecd3..0dfd9c7fe3515 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoFieldData.java @@ -20,13 +20,13 @@ /** - * {@link AtomicFieldData} specialization for geo points. + * {@link AtomicFieldData} specialization for geo points and shapes. */ -public interface AtomicGeoPointFieldData extends AtomicFieldData { +public interface AtomicGeoFieldData extends AtomicFieldData { /** - * Return geo-point values. + * Return geo values. */ - MultiGeoPointValues getGeoPointValues(); + MultiGeoValues getGeoValues(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 68b8f2c85325f..1fcfa26fb9fd1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -25,7 +25,8 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -69,27 +70,33 @@ public static SortedNumericDoubleValues emptySortedNumericDoubles() { return singleton(emptyNumericDouble()); } - public static GeoPointValues emptyGeoPoint() { - return new GeoPointValues() { + /** + * Return a {@link MultiGeoValues} that doesn't contain any value. + */ + public static MultiGeoValues emptyMultiGeoValues() { + return new MultiGeoValues() { @Override - public boolean advanceExact(int doc) throws IOException { + public boolean advanceExact(int doc) { return false; } @Override - public GeoPoint geoPointValue() { + public int docValueCount() { + return 0; + } + + @Override + public ValuesSourceType valuesSourceType() { + return CoreValuesSourceType.GEO; + } + + @Override + public GeoValue nextValue() { throw new UnsupportedOperationException(); } }; } - /** - * Return a {@link SortedNumericDoubleValues} that doesn't contain any value. - */ - public static MultiGeoPointValues emptyMultiGeoPoints() { - return singleton(emptyGeoPoint()); - } - /** * Returns a {@link DocValueBits} representing all documents from values that have a value. */ @@ -119,7 +126,7 @@ public boolean advanceExact(int doc) throws IOException { * Returns a {@link DocValueBits} representing all documents from pointValues that have * a value. */ - public static DocValueBits docsWithValue(final MultiGeoPointValues pointValues) { + public static DocValueBits docsWithValue(final MultiGeoValues pointValues) { return new DocValueBits() { @Override public boolean advanceExact(int doc) throws IOException { @@ -246,20 +253,13 @@ public static NumericDoubleValues unwrapSingleton(SortedNumericDoubleValues valu } /** - * Returns a multi-valued view over the provided {@link GeoPointValues}. - */ - public static MultiGeoPointValues singleton(GeoPointValues values) { - return new SingletonMultiGeoPointValues(values); - } - - /** - * Returns a single-valued view of the {@link MultiGeoPointValues}, - * if it was previously wrapped with {@link #singleton(GeoPointValues)}, + * Returns a single-valued view of the {@link MultiGeoValues}, + * if it was previously wrapped {@link SingletonMultiGeoPointValues}, * or null. */ - public static GeoPointValues unwrapSingleton(MultiGeoPointValues values) { + public static MultiGeoValues unwrapSingleton(MultiGeoValues values) { if (values instanceof SingletonMultiGeoPointValues) { - return ((SingletonMultiGeoPointValues) values).getGeoPointValues(); + return ((SingletonMultiGeoPointValues) values).getGeoValues(); } return null; } @@ -375,7 +375,7 @@ public BytesRef nextValue() throws IOException { * typically used for scripts or for the `map` execution mode of terms aggs. * NOTE: this is very slow! */ - public static SortedBinaryDocValues toString(final MultiGeoPointValues values) { + public static SortedBinaryDocValues toString(final MultiGeoValues values) { return toString(new ToStringValues() { @Override public boolean advanceExact(int doc) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java index 53466e9b4ec61..b44665f1e6f72 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java @@ -21,7 +21,7 @@ /** - * Specialization of {@link IndexFieldData} for geo points. + * Specialization of {@link IndexGeometryFieldData} for geo points. */ -public interface IndexGeoPointFieldData extends IndexFieldData { +public interface IndexGeoPointFieldData extends IndexGeometryFieldData { } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java new file mode 100644 index 0000000000000..c0a6bf5f06f32 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + + +/** + * Specialization of {@link IndexGeometryFieldData} for geo shapes. + */ +public interface IndexGeoShapeFieldData extends IndexGeometryFieldData { +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeometryFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeometryFieldData.java new file mode 100644 index 0000000000000..ef83bd51a957d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeometryFieldData.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + + +/** + * Specialization of {@link IndexFieldData} for geo points and shapes. + */ +public interface IndexGeometryFieldData extends IndexFieldData { +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java deleted file mode 100644 index c80c337c6d0e3..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata; - -import org.elasticsearch.common.geo.GeoPoint; - -import java.io.IOException; - -/** - * A stateful lightweight per document set of {@link GeoPoint} values. - * To iterate over values in a document use the following pattern: - * 
- *   GeoPointValues values = ..;
- *   values.setDocId(docId);
- *   final int numValues = values.count();
- *   for (int i = 0; i < numValues; i++) {
- *       GeoPoint value = values.valueAt(i);
- *       // process value
- *   }
- *
- * The set of values associated with a document might contain duplicates and - * comes in a non-specified order. - */ -public abstract class MultiGeoPointValues { - - /** - * Creates a new {@link MultiGeoPointValues} instance - */ - protected MultiGeoPointValues() { - } - - /** - * Advance this instance to the given document id - * @return true if there is a value for this document - */ - public abstract boolean advanceExact(int doc) throws IOException; - - /** - * Return the number of geo points the current document has. - */ - public abstract int docValueCount(); - - /** - * Return the next value associated with the current document. This must not be - * called more than {@link #docValueCount()} times. - * - * Note: the returned {@link GeoPoint} might be shared across invocations. - * - * @return the next value for the current docID set to {@link #advanceExact(int)}. - */ - public abstract GeoPoint nextValue() throws IOException; - -} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java new file mode 100644 index 0000000000000..e5bb77b4b20ca --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -0,0 +1,320 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.CentroidCalculator; +import org.elasticsearch.common.geo.CoordinateEncoder; +import org.elasticsearch.common.geo.DimensionalShapeType; +import org.elasticsearch.common.geo.Extent; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.TriangleTreeReader; +import org.elasticsearch.common.geo.TriangleTreeWriter; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +import java.io.IOException; +import java.text.ParseException; +import java.util.Arrays; +import java.util.List; + +/** + * A stateful lightweight per document set of geo values. + * To iterate over values in a document use the following pattern: + * 
+ *   MultiGeoValues values = ..;
+ *   values.setDocId(docId);
+ *   final int numValues = values.count();
+ *   for (int i = 0; i < numValues; i++) {
+ *       GeoValue value = values.valueAt(i);
+ *       // process value
+ *   }
+ *
+ * The set of values associated with a document might contain duplicates and + * comes in a non-specified order. + */ +public abstract class MultiGeoValues { + + /** + * Creates a new {@link MultiGeoValues} instance + */ + protected MultiGeoValues() { + } + + /** + * Advance this instance to the given document id + * @return true if there is a value for this document + */ + public abstract boolean advanceExact(int doc) throws IOException; + + /** + * Return the number of geo points the current document has. + */ + public abstract int docValueCount(); + + public abstract ValuesSourceType valuesSourceType(); + + /** + * Return the next value associated with the current document. This must not be + * called more than {@link #docValueCount()} times. + * + * Note: the returned {@link GeoValue} might be shared across invocations. + * + * @return the next value for the current docID set to {@link #advanceExact(int)}. + */ + public abstract GeoValue nextValue() throws IOException; + + public static class GeoPointValue implements GeoValue { + private final GeoPoint geoPoint; + private final BoundingBox boundingBox; + + public GeoPointValue(GeoPoint geoPoint) { + this.boundingBox = new BoundingBox(); + this.geoPoint = geoPoint; + } + + public GeoPoint geoPoint() { + return geoPoint; + } + + @Override + public BoundingBox boundingBox() { + boundingBox.reset(geoPoint); + return boundingBox; + } + + @Override + public GeoRelation relate(Rectangle rectangle) { + if (geoPoint.lat() >= rectangle.getMinLat() && geoPoint.lat() <= rectangle.getMaxLat() + && geoPoint.lon() >= rectangle.getMinLon() && geoPoint.lon() <= rectangle.getMaxLon()) { + return GeoRelation.QUERY_CROSSES; + } + return GeoRelation.QUERY_DISJOINT; + } + + @Override + public DimensionalShapeType dimensionalShapeType() { + return DimensionalShapeType.POINT; + } + + @Override + public double weight() { + return 1.0; + } + + @Override + public double lat() { + return geoPoint.lat(); + } + + @Override + public double lon() { + return geoPoint.lon(); + } + + @Override + public String toString() { + return geoPoint.toString(); + } + } + + public static class GeoShapeValue implements GeoValue { + private static final WellKnownText MISSING_GEOMETRY_PARSER = new WellKnownText(true, new GeographyValidator(true)); + + private final TriangleTreeReader reader; + private final BoundingBox boundingBox; + + public GeoShapeValue(TriangleTreeReader reader) { + this.reader = reader; + this.boundingBox = new BoundingBox(); + } + + @Override + public BoundingBox boundingBox() { + boundingBox.reset(reader.getExtent(), GeoShapeCoordinateEncoder.INSTANCE); + return boundingBox; + } + + /** + * @return the latitude of the centroid of the shape + */ + @Override + public GeoRelation relate(Rectangle rectangle) { + int minX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMinX()); + int maxX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMaxX()); + int minY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMinY()); + int maxY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMaxY()); + return reader.relateTile(minX, minY, maxX, maxY); + } + + @Override + public DimensionalShapeType dimensionalShapeType() { + return reader.getDimensionalShapeType(); + } + + @Override + public double weight() { + return reader.getSumCentroidWeight(); + } + + @Override + public double lat() { + return reader.getCentroidY(); + } + + /** + * @return the longitude of the centroid of the shape + */ + @Override + public double lon() { + return reader.getCentroidX(); + } + + public static GeoShapeValue missing(String missing) { + try { + Geometry geometry = MISSING_GEOMETRY_PARSER.fromWKT(missing); + ShapeField.DecodedTriangle[] triangles = toDecodedTriangles(geometry); + TriangleTreeWriter writer = + new TriangleTreeWriter(Arrays.asList(triangles), GeoShapeCoordinateEncoder.INSTANCE, + new CentroidCalculator(geometry)); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); + writer.writeTo(output); + TriangleTreeReader reader = new TriangleTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + reader.reset(new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size()))); + return new GeoShapeValue(reader); + } catch (IOException | ParseException e) { + throw new IllegalArgumentException("Can't apply missing value [" + missing + "]", e); + } + } + + private static ShapeField.DecodedTriangle[] toDecodedTriangles(Geometry geometry) { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + List fields = indexer.indexShape(null, geometry); + ShapeField.DecodedTriangle[] triangles = new ShapeField.DecodedTriangle[fields.size()]; + final byte[] scratch = new byte[7 * Integer.BYTES]; + for (int i = 0; i < fields.size(); i++) { + BytesRef bytesRef = fields.get(i).binaryValue(); + assert bytesRef.length == 7 * Integer.BYTES; + System.arraycopy(bytesRef.bytes, bytesRef.offset, scratch, 0, 7 * Integer.BYTES); + ShapeField.decodeTriangle(scratch, triangles[i] = new ShapeField.DecodedTriangle()); + } + return triangles; + } + } + + /** + * interface for geo-shape and geo-point doc-values to + * retrieve properties used in aggregations. + */ + public interface GeoValue { + double lat(); + double lon(); + BoundingBox boundingBox(); + GeoRelation relate(Rectangle rectangle); + DimensionalShapeType dimensionalShapeType(); + double weight(); + } + + public static class BoundingBox { + public double top; + public double bottom; + public double negLeft; + public double negRight; + public double posLeft; + public double posRight; + + private BoundingBox() { + } + + private void reset(Extent extent, CoordinateEncoder coordinateEncoder) { + this.top = coordinateEncoder.decodeY(extent.top); + this.bottom = coordinateEncoder.decodeY(extent.bottom); + + if (extent.negLeft == Integer.MAX_VALUE && extent.negRight == Integer.MIN_VALUE) { + this.negLeft = Double.POSITIVE_INFINITY; + this.negRight = Double.NEGATIVE_INFINITY; + } else { + this.negLeft = coordinateEncoder.decodeX(extent.negLeft); + this.negRight = coordinateEncoder.decodeX(extent.negRight); + } + + if (extent.posLeft == Integer.MAX_VALUE && extent.posRight == Integer.MIN_VALUE) { + this.posLeft = Double.POSITIVE_INFINITY; + this.posRight = Double.NEGATIVE_INFINITY; + } else { + this.posLeft = coordinateEncoder.decodeX(extent.posLeft); + this.posRight = coordinateEncoder.decodeX(extent.posRight); + } + } + + private void reset(GeoPoint point) { + this.top = point.lat(); + this.bottom = point.lat(); + if (point.lon() < 0) { + this.negLeft = point.lon(); + this.negRight = point.lon(); + this.posLeft = Double.POSITIVE_INFINITY; + this.posRight = Double.NEGATIVE_INFINITY; + } else { + this.negLeft = Double.POSITIVE_INFINITY; + this.negRight = Double.NEGATIVE_INFINITY; + this.posLeft = point.lon(); + this.posRight = point.lon(); + } + } + + /** + * @return the minimum y-coordinate of the extent + */ + public double minY() { + return bottom; + } + + /** + * @return the maximum y-coordinate of the extent + */ + public double maxY() { + return top; + } + + /** + * @return the absolute minimum x-coordinate of the extent, whether it is positive or negative. + */ + public double minX() { + return Math.min(negLeft, posLeft); + } + + /** + * @return the absolute maximum x-coordinate of the extent, whether it is positive or negative. + */ + public double maxX() { + return Math.max(negRight, posRight); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index a5349b5137996..e1ee6824cb64f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -260,11 +260,11 @@ public int size() { public static final class GeoPoints extends ScriptDocValues { - private final MultiGeoPointValues in; + private final MultiGeoValues in; private GeoPoint[] values = new GeoPoint[0]; private int count; - public GeoPoints(MultiGeoPointValues in) { + public GeoPoints(MultiGeoValues in) { this.in = in; } @@ -273,7 +273,7 @@ public void setNextDocId(int docId) throws IOException { if (in.advanceExact(docId)) { resize(in.docValueCount()); for (int i = 0; i < count; i++) { - GeoPoint point = in.nextValue(); + MultiGeoValues.GeoValue point = in.nextValue(); values[i] = new GeoPoint(point.lat(), point.lon()); } } else { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java index bae522e7b5081..6ce53ee72c3af 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java @@ -19,15 +19,16 @@ package org.elasticsearch.index.fielddata; -import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; -final class SingletonMultiGeoPointValues extends MultiGeoPointValues { +final class SingletonMultiGeoPointValues extends MultiGeoValues { - private final GeoPointValues in; + private final MultiGeoValues in; - SingletonMultiGeoPointValues(GeoPointValues in) { + SingletonMultiGeoPointValues(MultiGeoValues in) { this.in = in; } @@ -42,11 +43,16 @@ public int docValueCount() { } @Override - public GeoPoint nextValue() { - return in.geoPointValue(); + public ValuesSourceType valuesSourceType() { + return CoreValuesSourceType.GEOPOINT; } - GeoPointValues getGeoPointValues() { + @Override + public GeoValue nextValue() throws IOException { + return in.nextValue(); + } + + MultiGeoValues getGeoValues() { return in; } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java index 5d6575e43783c..3767d795e3547 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java @@ -19,28 +19,28 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.util.Accountable; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import java.util.Collection; import java.util.Collections; -public abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointFieldData { +public abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoFieldData { @Override public final SortedBinaryDocValues getBytesValues() { - return FieldData.toString(getGeoPointValues()); + return FieldData.toString(getGeoValues()); } @Override public final ScriptDocValues.GeoPoints getScriptValues() { - return new ScriptDocValues.GeoPoints(getGeoPointValues()); + return new ScriptDocValues.GeoPoints(getGeoValues()); } - public static AtomicGeoPointFieldData empty(final int maxDoc) { + public static AtomicGeoFieldData empty(final int maxDoc) { return new AbstractAtomicGeoPointFieldData() { @Override @@ -58,8 +58,8 @@ public void close() { } @Override - public MultiGeoPointValues getGeoPointValues() { - return FieldData.emptyMultiGeoPoints(); + public MultiGeoValues getGeoValues() { + return FieldData.emptyMultiGeoValues(); } }; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoShapeFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoShapeFieldData.java new file mode 100644 index 0000000000000..bb98623e84697 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoShapeFieldData.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.util.Accountable; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; + +import java.util.Collection; +import java.util.Collections; + +public abstract class AbstractAtomicGeoShapeFieldData implements AtomicGeoFieldData { + + @Override + public final SortedBinaryDocValues getBytesValues() { + return FieldData.toString(getGeoValues()); + } + + @Override + public final ScriptDocValues.BytesRefs getScriptValues() { + throw new UnsupportedOperationException(); + } + + public static AtomicGeoFieldData empty(final int maxDoc) { + return new AbstractAtomicGeoShapeFieldData() { + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() { + } + + @Override + public MultiGeoValues getGeoValues() { + return FieldData.emptyMultiGeoValues(); + } + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java index 93e57e5bcda08..9e766b63f9b5d 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java @@ -28,7 +28,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; @@ -41,8 +41,7 @@ import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; -public abstract class AbstractLatLonPointDVIndexFieldData extends DocValuesIndexFieldData - implements IndexGeoPointFieldData { +public abstract class AbstractLatLonPointDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoPointFieldData { AbstractLatLonPointDVIndexFieldData(Index index, String fieldName) { super(index, fieldName); } @@ -65,7 +64,7 @@ public LatLonPointDVIndexFieldData(Index index, String fieldName) { } @Override - public AtomicGeoPointFieldData load(LeafReaderContext context) { + public AtomicGeoFieldData load(LeafReaderContext context) { LeafReader reader = context.reader(); FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName); if (info != null) { @@ -75,7 +74,7 @@ public AtomicGeoPointFieldData load(LeafReaderContext context) { } @Override - public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { + public AtomicGeoFieldData loadDirect(LeafReaderContext context) throws Exception { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonShapeDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonShapeDVIndexFieldData.java new file mode 100644 index 0000000000000..a882e740f8332 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonShapeDVIndexFieldData.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +public abstract class AbstractLatLonShapeDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoShapeFieldData { + AbstractLatLonShapeDVIndexFieldData(Index index, String fieldName) { + super(index, fieldName); + } + + @Override + public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, + boolean reverse) { + throw new IllegalArgumentException("can't sort on geo_shape field without using specific sorting feature, like geo_distance"); + } + + public static class LatLonShapeDVIndexFieldData extends AbstractLatLonShapeDVIndexFieldData { + public LatLonShapeDVIndexFieldData(Index index, String fieldName) { + super(index, fieldName); + } + + @Override + public AtomicGeoFieldData load(LeafReaderContext context) { + LeafReader reader = context.reader(); + FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName); + if (info != null) { + checkCompatible(info); + } + return new LatLonShapeDVAtomicFieldData(reader, fieldName); + } + + @Override + public AtomicGeoFieldData loadDirect(LeafReaderContext context) throws Exception { + return load(context); + } + + @Override + public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, SortOrder sortOrder, DocValueFormat format, + int bucketSize, BucketedSort.ExtraData extra) { + throw new IllegalArgumentException("can't sort on geo_shape field without using specific sorting feature, like geo_distance"); + } + + /** helper: checks a fieldinfo and throws exception if its definitely not a LatLonDocValuesField */ + static void checkCompatible(FieldInfo fieldInfo) { + // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment. + if (fieldInfo.getDocValuesType() != DocValuesType.NONE + && fieldInfo.getDocValuesType() != DocValuesType.BINARY) { + throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + + fieldInfo.getDocValuesType() + " but this type has docValuesType=" + + DocValuesType.BINARY + ", is the field really a geo-shape field?"); + } + } + } + + public static class Builder implements IndexFieldData.Builder { + @Override + public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, + CircuitBreakerService breakerService, MapperService mapperService) { + // ignore breaker + return new LatLonShapeDVIndexFieldData(indexSettings.getIndex(), fieldType.name()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java index dba7dfb0c9e99..8ac4de271567c 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java @@ -24,7 +24,9 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.Accountable; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Collection; @@ -56,12 +58,12 @@ public void close() { } @Override - public MultiGeoPointValues getGeoPointValues() { + public MultiGeoValues getGeoValues() { try { final SortedNumericDocValues numericValues = DocValues.getSortedNumeric(reader, fieldName); - return new MultiGeoPointValues() { + return new MultiGeoValues() { - final GeoPoint point = new GeoPoint(); + final GeoPointValue point = new GeoPointValue(new GeoPoint()); @Override public boolean advanceExact(int doc) throws IOException { @@ -74,9 +76,14 @@ public int docValueCount() { } @Override - public GeoPoint nextValue() throws IOException { + public ValuesSourceType valuesSourceType() { + return CoreValuesSourceType.GEOPOINT; + } + + @Override + public GeoValue nextValue() throws IOException { final long encoded = numericValues.nextValue(); - point.reset(GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)), + point.geoPoint().reset(GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)), GeoEncodingUtils.decodeLongitude((int) encoded)); return point; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java new file mode 100644 index 0000000000000..51cc6cdd53765 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java @@ -0,0 +1,95 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.TriangleTreeReader; +import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +final class LatLonShapeDVAtomicFieldData extends AbstractAtomicGeoShapeFieldData { + private final LeafReader reader; + private final String fieldName; + + LatLonShapeDVAtomicFieldData(LeafReader reader, String fieldName) { + super(); + this.reader = reader; + this.fieldName = fieldName; + } + + @Override + public long ramBytesUsed() { + return 0; // not exposed by lucene + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() { + // noop + } + + @Override + public MultiGeoValues getGeoValues() { + try { + final BinaryDocValues binaryValues = DocValues.getBinary(reader, fieldName); + final TriangleTreeReader reader = new TriangleTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + final MultiGeoValues.GeoShapeValue geoShapeValue = new MultiGeoValues.GeoShapeValue(reader); + return new MultiGeoValues() { + + @Override + public boolean advanceExact(int doc) throws IOException { + return binaryValues.advanceExact(doc); + } + + @Override + public int docValueCount() { + return 1; + } + + @Override + public ValuesSourceType valuesSourceType() { + return CoreValuesSourceType.GEOSHAPE; + } + + @Override + public GeoValue nextValue() throws IOException { + final BytesRef encoded = binaryValues.binaryValue(); + reader.reset(encoded); + return geoShapeValue; + } + }; + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 006af39b8f59b..7ef03a8632aa1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -18,14 +18,17 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.ShapeField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; @@ -67,6 +70,7 @@ public static class Defaults { public static final Explicit COERCE = new Explicit<>(false, false); public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); + public static final Explicit DOC_VALUES = new Explicit<>(false, false); } @@ -80,6 +84,9 @@ public interface Indexer { Class processedClass(); List indexShape(ParseContext context, Processed shape); + + void indexDocValueField(ParseContext context, ShapeField.DecodedTriangle[] triangles, + CentroidCalculator centroidCalculator); } /** @@ -116,15 +123,6 @@ public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFi super(name, fieldType, defaultFieldType); } - public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType, - boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ) { - super(name, fieldType, defaultFieldType); - this.coerce = coerce; - this.ignoreMalformed = ignoreMalformed; - this.orientation = orientation; - this.ignoreZValue = ignoreZ; - } - public Builder coerce(boolean coerce) { this.coerce = coerce; return this; @@ -184,6 +182,15 @@ public Builder ignoreZValue(final boolean ignoreZValue) { return this; } + protected Explicit docValues() { + if (docValuesSet && fieldType.hasDocValues()) { + return new Explicit<>(true, true); + } else if (docValuesSet) { + return new Explicit<>(false, true); + } + return Defaults.DOC_VALUES; + } + @Override protected void setupFieldType(BuilderContext context) { super.setupFieldType(context); @@ -245,6 +252,9 @@ public Mapper.Builder parse(String name, Map node, ParserContext XContentMapValues.nodeBooleanValue(fieldNode, name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName())); iterator.remove(); + } else if (TypeParsers.DOC_VALUES.equals(fieldName)) { + params.put(TypeParsers.DOC_VALUES, XContentMapValues.nodeBooleanValue(fieldNode, name + "." + TypeParsers.DOC_VALUES)); + iterator.remove(); } } if (parsedDeprecatedParameters == false) { @@ -252,6 +262,10 @@ public Mapper.Builder parse(String name, Map node, ParserContext } Builder builder = newBuilder(name, params); + if (params.containsKey(TypeParsers.DOC_VALUES)) { + builder.docValues((Boolean) params.get(TypeParsers.DOC_VALUES)); + } + if (params.containsKey(Names.COERCE.getPreferredName())) { builder.coerce((Boolean)params.get(Names.COERCE.getPreferredName())); } @@ -352,15 +366,17 @@ public QueryProcessor geometryQueryBuilder() { protected Explicit coerce; protected Explicit ignoreMalformed; protected Explicit ignoreZValue; + protected Explicit docValues; protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, - Explicit ignoreZValue, Settings indexSettings, + Explicit ignoreZValue, Explicit docValues, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); this.coerce = coerce; this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; + this.docValues = docValues; } @Override @@ -376,6 +392,9 @@ protected void doMerge(Mapper mergeWith) { if (gsfm.ignoreZValue.explicit()) { this.ignoreZValue = gsfm.ignoreZValue; } + if (gsfm.docValues.explicit()) { + this.docValues = gsfm.docValues; + } } @Override @@ -399,6 +418,9 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par if (includeDefaults || ignoreZValue.explicit()) { builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); } + if (includeDefaults || docValues.explicit()) { + builder.field(TypeParsers.DOC_VALUES, docValues.value()); + } } public Explicit coerce() { @@ -413,6 +435,10 @@ public Explicit ignoreZValue() { return ignoreZValue; } + public Explicit docValues() { + return docValues; + } + public Orientation orientation() { return ((AbstractGeometryFieldType)fieldType).orientation(); } @@ -434,8 +460,20 @@ public void parse(ParseContext context) throws IOException { shape = geometryIndexer.prepareForIndexing(geometry); } - List fields = new ArrayList<>(); - fields.addAll(geometryIndexer.indexShape(context, shape)); + List fields = new ArrayList<>(geometryIndexer.indexShape(context, shape)); + final byte[] scratch = new byte[7 * Integer.BYTES]; + if (fieldType().hasDocValues()) { + // doc values are generated from the indexed fields. + ShapeField.DecodedTriangle[] triangles = new ShapeField.DecodedTriangle[fields.size()]; + for (int i = 0; i < fields.size(); i++) { + BytesRef bytesRef = fields.get(i).binaryValue(); + assert bytesRef.length == 7 * Integer.BYTES; + System.arraycopy(bytesRef.bytes, bytesRef.offset, scratch, 0, 7 * Integer.BYTES); + ShapeField.decodeTriangle(scratch, triangles[i] = new ShapeField.DecodedTriangle()); + } + + geometryIndexer.indexDocValueField(context, triangles, new CentroidCalculator((Geometry) shape)); + } createFieldNamesField(context, fields); for (IndexableField field : fields) { context.doc().add(field); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java new file mode 100644 index 0000000000000..10884ba41ee41 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.geo.CentroidCalculator; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.TriangleTreeWriter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class BinaryGeoShapeDocValuesField extends CustomDocValuesField { + + private final List triangles; + private final CentroidCalculator centroidCalculator; + + public BinaryGeoShapeDocValuesField(String name, ShapeField.DecodedTriangle[] triangles, CentroidCalculator centroidCalculator) { + super(name); + this.triangles = new ArrayList<>(triangles.length); + this.centroidCalculator = centroidCalculator; + this.triangles.addAll(Arrays.asList(triangles)); + } + + public void add(ShapeField.DecodedTriangle[] triangles, CentroidCalculator centroidCalculator) { + this.triangles.addAll(Arrays.asList(triangles)); + this.centroidCalculator.addFrom(centroidCalculator); + } + + @Override + public BytesRef binaryValue() { + try { + final TriangleTreeWriter writer = new TriangleTreeWriter(triangles, GeoShapeCoordinateEncoder.INSTANCE, centroidCalculator); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); + writer.writeTo(output); + return new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size())); + } catch (IOException e) { + throw new ElasticsearchException("failed to encode shape", e); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index a8015d3508701..320c41284caad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -19,11 +19,20 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.AbstractLatLonShapeDVIndexFieldData; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor; /** @@ -58,10 +67,24 @@ public Builder(String name) { public GeoShapeFieldMapper build(BuilderContext context) { setupFieldType(context); return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), - ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + ignoreZValue(), docValues(), context.indexSettings(), + multiFieldsBuilder.build(this, context), copyTo); } @Override + public boolean defaultDocValues(Version indexCreated) { + return Version.V_8_0_0.onOrBefore(indexCreated); + } + + protected Explicit docValues() { + if (docValuesSet && fieldType.hasDocValues()) { + return new Explicit<>(true, true); + } else if (docValuesSet) { + return new Explicit<>(false, true); + } + return new Explicit<>(fieldType.hasDocValues(), false); + } + protected void setupFieldType(BuilderContext context) { super.setupFieldType(context); @@ -85,6 +108,26 @@ protected GeoShapeFieldType(GeoShapeFieldType ref) { super(ref); } + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + failIfNoDocValues(); + return new AbstractLatLonShapeDVIndexFieldData.Builder(); + } + + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: [" + + name() + "]"); + } + @Override public GeoShapeFieldType clone() { return new GeoShapeFieldType(this); @@ -98,9 +141,9 @@ public String typeName() { public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, - Explicit ignoreZValue, Settings indexSettings, + Explicit ignoreZValue, Explicit docValues, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, indexSettings, multiFields, copyTo); } @@ -124,4 +167,5 @@ public GeoShapeFieldType fieldType() { protected String contentType() { return CONTENT_TYPE; } + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java index 558b4068ffa90..725ee3202425f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java @@ -21,7 +21,9 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.document.ShapeField; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoLineDecomposer; import org.elasticsearch.common.geo.GeoPolygonDecomposer; import org.elasticsearch.geometry.Circle; @@ -185,6 +187,18 @@ public List indexShape(ParseContext context, Geometry shape) { return visitor.fields(); } + @Override + public void indexDocValueField(ParseContext context, ShapeField.DecodedTriangle[] triangles, CentroidCalculator calculator) { + BinaryGeoShapeDocValuesField docValuesField = + (BinaryGeoShapeDocValuesField) context.doc().getByKey(name); + if (docValuesField == null) { + docValuesField = new BinaryGeoShapeDocValuesField(name, triangles, calculator); + context.doc().addWithKey(name, docValuesField); + } else { + docValuesField.add(triangles, calculator); + } + } + private static class LuceneGeometryIndexer implements GeometryVisitor { private List fields = new ArrayList<>(); private String name; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 6fea1efaedafe..3fc5ac28c7b49 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -197,6 +197,16 @@ public GeoShapeFieldType fieldType() { return (GeoShapeFieldType)fieldType; } + public Builder docValues(boolean hasDocValues) { + super.docValues(hasDocValues); + if (hasDocValues) { + throw new ElasticsearchParseException("geo_shape field [" + name + + "] indexed using prefix-trees do not support doc_values"); + } + // doc-values already set to `false` + return this; + } + private void setupFieldTypeDeprecatedParameters(BuilderContext context) { GeoShapeFieldType ft = fieldType(); if (deprecatedParameters.strategy != null) { @@ -292,7 +302,7 @@ public LegacyGeoShapeFieldMapper build(BuilderContext context) { setupFieldType(context); return new LegacyGeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), - coerce(context), orientation(), ignoreZValue(), context.indexSettings(), + coerce(context), orientation(), ignoreZValue(), docValues(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } } @@ -318,6 +328,7 @@ public GeoShapeFieldType() { setStored(false); setStoreTermVectors(false); setOmitNorms(true); + setHasDocValues(false); } protected GeoShapeFieldType(GeoShapeFieldType ref) { @@ -470,10 +481,10 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, Explicit orientation, - Explicit ignoreZValue, Settings indexSettings, + Explicit ignoreZValue, Explicit docValues, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, - multiFields, copyTo); + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, + indexSettings, multiFields, copyTo); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeIndexer.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeIndexer.java index c751dbf3a28d5..81720b0daba4e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeIndexer.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeIndexer.java @@ -19,7 +19,9 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.ShapeField; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.XShapeCollection; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.locationtech.spatial4j.shape.Point; @@ -69,4 +71,9 @@ public List indexShape(ParseContext context, Shape shape) { } return Arrays.asList(fieldType.defaultPrefixTreeStrategy().createIndexableFields(shape)); } + + @Override + public void indexDocValueField(ParseContext context, ShapeField.DecodedTriangle[] triangles, CentroidCalculator centroidCalculator) { + throw new UnsupportedOperationException("doc values not supported for legacy shape indexer"); + } } diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 0b4177490b17b..87a020f867dd0 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -43,7 +43,7 @@ import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; @@ -354,7 +354,7 @@ public boolean needsScores() { @Override protected NumericDoubleValues distance(LeafReaderContext context) { - final MultiGeoPointValues geoPointValues = fieldData.load(context).getGeoPointValues(); + final MultiGeoValues geoPointValues = fieldData.load(context).getGeoValues(); return FieldData.replaceMissing(mode.select(new SortingNumericDoubleValues() { @Override public boolean advanceExact(int docId) throws IOException { @@ -362,7 +362,7 @@ public boolean advanceExact(int docId) throws IOException { int n = geoPointValues.docValueCount(); resize(n); for (int i = 0; i < n; i++) { - GeoPoint other = geoPointValues.nextValue(); + MultiGeoValues.GeoValue other = geoPointValues.nextValue(); double distance = distFunction.calculate( origin.lat(), origin.lon(), other.lat(), other.lon(), DistanceUnit.METERS); values[i] = Math.max(0.0d, distance - offset); @@ -380,11 +380,11 @@ public boolean advanceExact(int docId) throws IOException { protected String getDistanceString(LeafReaderContext ctx, int docId) throws IOException { StringBuilder values = new StringBuilder(mode.name()); values.append(" of: ["); - final MultiGeoPointValues geoPointValues = fieldData.load(ctx).getGeoPointValues(); + final MultiGeoValues geoPointValues = fieldData.load(ctx).getGeoValues(); if (geoPointValues.advanceExact(docId)) { final int num = geoPointValues.docValueCount(); for (int i = 0; i < num; i++) { - GeoPoint value = geoPointValues.nextValue(); + MultiGeoValues.GeoValue value = geoPointValues.nextValue(); values.append("Math.max(arcDistance("); values.append(value).append("(=doc value),"); values.append(origin).append("(=origin)) - ").append(offset).append("(=offset), 0)"); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index 165a50db60d18..b47ae3029bffd 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -31,8 +31,11 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler; import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridTiler; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridTiler; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; @@ -131,17 +134,26 @@ public boolean equals(Object obj) { protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { ValuesSource orig = config.toValuesSource(queryShardContext); if (orig == null) { - orig = ValuesSource.GeoPoint.EMPTY; + orig = ValuesSource.Geo.EMPTY; } - if (orig instanceof ValuesSource.GeoPoint) { - ValuesSource.GeoPoint geoPoint = (ValuesSource.GeoPoint) orig; + if (orig instanceof ValuesSource.Geo) { + ValuesSource.Geo geoValue = (ValuesSource.Geo) orig; // is specified in the builder. final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null; - CellIdSource cellIdSource = new CellIdSource(geoPoint, precision, geoBoundingBox, GeoTileUtils::longEncode); + + final GeoGridTiler tiler; + if (geoBoundingBox.isUnbounded()) { + tiler = new GeoTileGridTiler(); + } else { + tiler = new BoundedGeoTileGridTiler(geoBoundingBox); + } + + CellIdSource cellIdSource = new CellIdSource(geoValue, precision, tiler); return new CompositeValuesSourceConfig(name, fieldType, cellIdSource, DocValueFormat.GEOTILE, order(), missingBucket(), script() != null); } else { - throw new IllegalArgumentException("invalid source, expected geo_point, got " + orig.getClass().getSimpleName()); + throw new IllegalArgumentException("invalid source, expected one of [geo_point, geo_shape], got " + + orig.getClass().getSimpleName()); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/AllCellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/AllCellValues.java new file mode 100644 index 0000000000000..c9ee68c044f9e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/AllCellValues.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +import java.io.IOException; + +/** Sorted numeric doc values for precision 0 */ +class AllCellValues extends AbstractSortingNumericDocValues { + private MultiGeoValues geoValues; + + protected AllCellValues(MultiGeoValues geoValues, GeoGridTiler tiler) { + this.geoValues = geoValues; + resize(1); + values[0] = tiler.encode(0, 0, 0); + } + + // for testing + protected long[] getValues() { + return values; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + resize(1); + return geoValues.advanceExact(docId); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedCellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedCellValues.java deleted file mode 100644 index 52fddb34b0576..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedCellValues.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.aggregations.bucket.geogrid; - -import org.elasticsearch.common.geo.GeoBoundingBox; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; - -/** - * Class representing {@link CellValues} whose values are filtered - * according to whether they are within the specified {@link GeoBoundingBox}. - * - * The specified bounding box is assumed to be bounded. - */ -class BoundedCellValues extends CellValues { - - private final GeoBoundingBox geoBoundingBox; - - protected BoundedCellValues(MultiGeoPointValues geoValues, int precision, CellIdSource.GeoPointLongEncoder encoder, - GeoBoundingBox geoBoundingBox) { - super(geoValues, precision, encoder); - this.geoBoundingBox = geoBoundingBox; - } - - - @Override - int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) { - if (geoBoundingBox.pointInBounds(target.getLon(), target.getLat())) { - values[valuesIdx] = encoder.encode(target.getLon(), target.getLat(), precision); - return valuesIdx + 1; - } - return valuesIdx; - } -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java new file mode 100644 index 0000000000000..0636de647a22c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +public class BoundedGeoHashGridTiler extends GeoHashGridTiler { + private final double boundsTop; + private final double boundsBottom; + private final double boundsWestLeft; + private final double boundsWestRight; + private final double boundsEastLeft; + private final double boundsEastRight; + private final boolean crossesDateline; + + BoundedGeoHashGridTiler(GeoBoundingBox geoBoundingBox) { + // split geoBoundingBox into west and east boxes + boundsTop = geoBoundingBox.top(); + boundsBottom = geoBoundingBox.bottom(); + if (geoBoundingBox.right() < geoBoundingBox.left()) { + boundsWestLeft = -180; + boundsWestRight = geoBoundingBox.right(); + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = geoBoundingBox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + } + + @Override + public int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + long hash = encode(x, y, precision); + if (cellIntersectsGeoBoundingBox(Geohash.toBoundingBox(Geohash.stringEncode(hash)))) { + values[valuesIdx] = hash; + return valuesIdx + 1; + } + return valuesIdx; + } + + boolean cellIntersectsGeoBoundingBox(Rectangle rectangle) { + return (boundsTop >= rectangle.getMinY() && boundsBottom <= rectangle.getMaxY() + && (boundsEastLeft <= rectangle.getMaxX() && boundsEastRight >= rectangle.getMinX() + || (crossesDateline && boundsWestLeft <= rectangle.getMaxX() && boundsWestRight >= rectangle.getMinX()))); + } + + @Override + protected int setValue(CellValues docValues, MultiGeoValues.GeoValue geoValue, MultiGeoValues.BoundingBox bounds, int precision) { + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + GeoRelation relation = relateTile(geoValue, hash); + if (relation != GeoRelation.QUERY_DISJOINT) { + docValues.resizeCell(1); + docValues.add(0, Geohash.longEncode(hash)); + return 1; + } + return 0; + } + + @Override + protected GeoRelation relateTile(MultiGeoValues.GeoValue geoValue, String hash) { + Rectangle rectangle = Geohash.toBoundingBox(hash); + if (cellIntersectsGeoBoundingBox(rectangle)) { + return geoValue.relate(rectangle); + } else { + return GeoRelation.QUERY_DISJOINT; + } + } + + @Override + protected int setValuesForFullyContainedTile(String hash, CellValues values, + int valuesIndex, int targetPrecision) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + if (hashes[i].length() == targetPrecision ) { + if (cellIntersectsGeoBoundingBox(Geohash.toBoundingBox(hashes[i]))) { + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } + } else { + valuesIndex = setValuesForFullyContainedTile(hashes[i], values, valuesIndex, targetPrecision); + } + } + return valuesIndex; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java new file mode 100644 index 0000000000000..89b6b69c41ba7 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +public class BoundedGeoTileGridTiler extends GeoTileGridTiler { + private final double boundsTop; + private final double boundsBottom; + private final double boundsWestLeft; + private final double boundsWestRight; + private final double boundsEastLeft; + private final double boundsEastRight; + private final boolean crossesDateline; + + public BoundedGeoTileGridTiler(GeoBoundingBox geoBoundingBox) { + // split geoBoundingBox into west and east boxes + boundsTop = geoBoundingBox.top(); + boundsBottom = geoBoundingBox.bottom(); + if (geoBoundingBox.right() < geoBoundingBox.left()) { + boundsWestLeft = -180; + boundsWestRight = geoBoundingBox.right(); + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = geoBoundingBox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + } + + public int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + long hash = encode(x, y, precision); + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(hash))) { + values[valuesIdx] = hash; + return valuesIdx + 1; + } + return valuesIdx; + } + + boolean cellIntersectsGeoBoundingBox(Rectangle rectangle) { + return (boundsTop >= rectangle.getMinY() && boundsBottom <= rectangle.getMaxY() + && (boundsEastLeft <= rectangle.getMaxX() && boundsEastRight >= rectangle.getMinX() + || (crossesDateline && boundsWestLeft <= rectangle.getMaxX() && boundsWestRight >= rectangle.getMinX()))); + } + + @Override + public GeoRelation relateTile(MultiGeoValues.GeoValue geoValue, int xTile, int yTile, int precision) { + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + if (cellIntersectsGeoBoundingBox(rectangle)) { + return geoValue.relate(rectangle); + } + return GeoRelation.QUERY_DISJOINT; + } + + @Override + protected int setValue(CellValues docValues, MultiGeoValues.GeoValue geoValue, int xTile, int yTile, int precision) { + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(xTile, yTile, precision))) { + docValues.resizeCell(1); + docValues.add(0, GeoTileUtils.longEncodeTiles(precision, xTile, yTile)); + return 1; + } + return 0; + } + + @Override + protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, CellValues values, int valuesIndex, + int targetPrecision) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + if (zTile == targetPrecision) { + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(nextX, nextY, zTile))) { + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } + } else { + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java index 84f963bbbd9ce..8267bc69f8c52 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java @@ -20,27 +20,26 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; -import org.elasticsearch.common.geo.GeoBoundingBox; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; /** - * Wrapper class to help convert {@link MultiGeoPointValues} + * Wrapper class to help convert {@link MultiGeoValues} * to numeric long values for bucketing. */ public class CellIdSource extends ValuesSource.Numeric { - private final ValuesSource.GeoPoint valuesSource; + private final ValuesSource.Geo valuesSource; private final int precision; - private final GeoPointLongEncoder encoder; - private final GeoBoundingBox geoBoundingBox; + private final GeoGridTiler encoder; - public CellIdSource(GeoPoint valuesSource,int precision, GeoBoundingBox geoBoundingBox, GeoPointLongEncoder encoder) { + public CellIdSource(ValuesSource.Geo valuesSource, int precision, GeoGridTiler encoder) { this.valuesSource = valuesSource; //different GeoPoints could map to the same or different hashing cells. this.precision = precision; - this.geoBoundingBox = geoBoundingBox; this.encoder = encoder; } @@ -55,10 +54,21 @@ public boolean isFloatingPoint() { @Override public SortedNumericDocValues longValues(LeafReaderContext ctx) { - if (geoBoundingBox.isUnbounded()) { - return new UnboundedCellValues(valuesSource.geoPointValues(ctx), precision, encoder); + MultiGeoValues geoValues = valuesSource.geoValues(ctx); + if (precision == 0) { + // special case, precision 0 is the whole world + return new AllCellValues(geoValues, encoder); + } + ValuesSourceType vs = geoValues.valuesSourceType(); + if (CoreValuesSourceType.GEOPOINT == vs) { + // docValues are geo points + return new GeoPointCellValues(geoValues, precision, encoder); + } else if (CoreValuesSourceType.GEOSHAPE == vs || CoreValuesSourceType.GEO == vs) { + // docValues are geo shapes + return new GeoShapeCellValues(geoValues, precision, encoder); + } else { + throw new IllegalArgumentException("unsupported geo type"); } - return new BoundedCellValues(valuesSource.geoPointValues(ctx), precision, encoder, geoBoundingBox); } @Override @@ -70,14 +80,4 @@ public SortedNumericDoubleValues doubleValues(LeafReaderContext ctx) { public SortedBinaryDocValues bytesValues(LeafReaderContext ctx) { throw new UnsupportedOperationException(); } - - /** - * The encoder to use to convert a geopoint's (lon, lat, precision) into - * a long-encoded bucket key for aggregating. - */ - @FunctionalInterface - public interface GeoPointLongEncoder { - long encode(double lon, double lat, int precision); - } - } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellValues.java index 5d428373ccd8f..6ebad166f513d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellValues.java @@ -19,7 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import java.io.IOException; @@ -29,14 +29,14 @@ * sort them in order to account for the cells correctly. */ abstract class CellValues extends AbstractSortingNumericDocValues { - private MultiGeoPointValues geoValues; + private MultiGeoValues geoValues; protected int precision; - protected CellIdSource.GeoPointLongEncoder encoder; + protected GeoGridTiler tiler; - protected CellValues(MultiGeoPointValues geoValues, int precision, CellIdSource.GeoPointLongEncoder encoder) { + protected CellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { this.geoValues = geoValues; this.precision = precision; - this.encoder = encoder; + this.tiler = tiler; } @Override @@ -56,6 +56,19 @@ public boolean advanceExact(int docId) throws IOException { } } + // for testing + protected long[] getValues() { + return values; + } + + protected void add(int idx, long value) { + values[idx] = value; + } + + void resizeCell(int newSize) { + resize(newSize); + } + /** * Sets the appropriate long-encoded value for target * in values. @@ -64,5 +77,6 @@ public boolean advanceExact(int docId) throws IOException { * @param valuesIdx the index into values to set * @return valuesIdx + 1 if value was set, valuesIdx otherwise. */ - abstract int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx); + abstract int advanceValue(MultiGeoValues.GeoValue target, int valuesIdx); + } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index 0a7c918231c73..d57ff9ec71043 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -46,7 +46,7 @@ import java.util.Map; import java.util.Objects; -public abstract class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder +public abstract class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder { /* recognized field names in JSON */ static final ParseField FIELD_PRECISION = new ParseField("precision"); @@ -79,7 +79,7 @@ public static ObjectParser createParser(String } public GeoGridAggregationBuilder(String name) { - super(name, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(name, CoreValuesSourceType.GEO, ValueType.GEO); } protected GeoGridAggregationBuilder(GeoGridAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -94,7 +94,7 @@ protected GeoGridAggregationBuilder(GeoGridAggregationBuilder clone, Builder fac * Read from a stream. */ public GeoGridAggregationBuilder(StreamInput in) throws IOException { - super(in, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(in, CoreValuesSourceType.GEO, ValueType.GEO); precision = in.readVInt(); requiredSize = in.readVInt(); shardSize = in.readVInt(); @@ -123,10 +123,10 @@ protected void innerWriteTo(StreamOutput out) throws IOException { /** * Creates a new instance of the {@link ValuesSourceAggregatorFactory}-derived class specific to the geo aggregation. */ - protected abstract ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, - GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, - Builder subFactoriesBuilder, Map metaData + protected abstract ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, Builder subFactoriesBuilder, + Map metaData ) throws IOException; public int precision() { @@ -170,8 +170,8 @@ public GeoBoundingBox geoBoundingBox() { } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { int shardSize = this.shardSize; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java new file mode 100644 index 0000000000000..1f98f9bdf3031 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.index.fielddata.MultiGeoValues; + +/** + * The tiler to use to convert a geo value into long-encoded bucket keys for aggregating. + */ +public interface GeoGridTiler { + /** + * encodes a single point to its long-encoded bucket key value. + * + * @param x the x-coordinate + * @param y the y-coordinate + * @param precision the zoom level of tiles + */ + long encode(double x, double y, int precision); + + /** + * + * @param docValues the array of long-encoded bucket keys to fill + * @param geoValue the input shape + * @param precision the tile zoom-level + * + * @return the number of tiles the geoValue intersects + */ + int setValues(CellValues docValues, MultiGeoValues.GeoValue geoValue, int precision); + + + /** + * This sets the long-encoded value of the geo-point into the associated doc-values + * array. This is to be overridden by the {@link BoundedGeoTileGridTiler} and + * {@link BoundedGeoHashGridTiler} to check whether the point's tile intersects + * the appropriate bounds. + * + * @param values the doc-values array + * @param x the longitude of the point + * @param y the latitude of the point + * @param precision the zoom-level + * @param valuesIdx the index into the doc-values array at the time of advancement + * + * @return the next index into the array + */ + default int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + values[valuesIdx] = encode(x, y, precision); + return valuesIdx + 1; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java index acc9cde113164..cf522280aa3cf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java @@ -60,13 +60,13 @@ public GeoGridAggregationBuilder precision(int precision) { } @Override - protected ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, - GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, - AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, - Map metaData) throws IOException { - return new GeoHashGridAggregatorFactory(name, config, precision, requiredSize, shardSize, geoBoundingBox, - queryShardContext, parent, subFactoriesBuilder, metaData); + protected ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metaData) throws IOException { + return new GeoHashGridAggregatorFactory(name, config, precision, requiredSize, shardSize, geoBoundingBox, queryShardContext, parent, + subFactoriesBuilder, metaData); } private GeoHashGridAggregationBuilder(GeoHashGridAggregationBuilder clone, AggregatorFactories.Builder factoriesBuilder, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java index 2d7087a693bfa..0999732dc31d9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java @@ -20,7 +20,6 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.common.geo.GeoBoundingBox; -import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -38,14 +37,14 @@ import java.util.List; import java.util.Map; -public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory { +public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory { private final int precision; private final int requiredSize; private final int shardSize; private final GeoBoundingBox geoBoundingBox; - GeoHashGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, + GeoHashGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { @@ -72,7 +71,7 @@ public InternalAggregation buildEmptyAggregation() { } @Override - protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(final ValuesSource.Geo valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, @@ -81,7 +80,13 @@ protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, geoBoundingBox, Geohash::longEncode); + final GeoGridTiler tiler; + if (geoBoundingBox.isUnbounded()) { + tiler = new GeoHashGridTiler(); + } else { + tiler = new BoundedGeoHashGridTiler(geoBoundingBox); + } + CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, tiler); return new GeoHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, searchContext, parent, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTiler.java new file mode 100644 index 0000000000000..7e60e7752d58c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTiler.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +public class GeoHashGridTiler implements GeoGridTiler { + + @Override + public long encode(double x, double y, int precision) { + return Geohash.longEncode(x, y, precision); + } + + @Override + public int setValues(CellValues values, MultiGeoValues.GeoValue geoValue, int precision) { + if (precision == 1) { + values.resizeCell(1); + values.add(0, Geohash.longEncode(0, 0, 0)); + } + + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + assert bounds.minX() <= bounds.maxX(); + long numLonCells = (long) ((bounds.maxX() - bounds.minX()) / Geohash.lonWidthInDegrees(precision)); + long numLatCells = (long) ((bounds.maxY() - bounds.minY()) / Geohash.latHeightInDegrees(precision)); + long count = (numLonCells + 1) * (numLatCells + 1); + if (count == 1) { + return setValue(values, geoValue, bounds, precision); + } else if (count <= precision) { + return setValuesByBruteForceScan(values, geoValue, precision, bounds); + } else { + return setValuesByRasterization("", values, 0, precision, geoValue); + } + } + + /** + * Sets a singular doc-value for the {@link MultiGeoValues.GeoValue}. To be overriden by {@link BoundedGeoHashGridTiler} + * to account for {@link org.elasticsearch.common.geo.GeoBoundingBox} conditions + */ + protected int setValue(CellValues docValues, MultiGeoValues.GeoValue geoValue, MultiGeoValues.BoundingBox bounds, int precision) { + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + docValues.resizeCell(1); + docValues.add(0, Geohash.longEncode(hash)); + return 1; + } + + protected GeoRelation relateTile(MultiGeoValues.GeoValue geoValue, String hash) { + Rectangle rectangle = Geohash.toBoundingBox(hash); + return geoValue.relate(rectangle); + } + + protected int setValuesByBruteForceScan(CellValues values, MultiGeoValues.GeoValue geoValue, int precision, + MultiGeoValues.BoundingBox bounds) { + // TODO: This way to discover cells inside of a bounding box seems not to work as expected. I can + // see that eventually we will be visiting twice the same cell which should not happen. + int idx = 0; + String min = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + String max = Geohash.stringEncode(bounds.maxX(), bounds.maxY(), precision); + String minNeighborBelow = Geohash.getNeighbor(min, precision, 0, -1); + double minY = Geohash.decodeLatitude((minNeighborBelow == null) ? min : minNeighborBelow); + double minX = Geohash.decodeLongitude(min); + double maxY = Geohash.decodeLatitude(max); + double maxX = Geohash.decodeLongitude(max); + for (double i = minX; i <= maxX; i += Geohash.lonWidthInDegrees(precision)) { + for (double j = minY; j <= maxY; j += Geohash.latHeightInDegrees(precision)) { + String hash = Geohash.stringEncode(i, j, precision); + GeoRelation relation = relateTile(geoValue, hash); + if (relation != GeoRelation.QUERY_DISJOINT) { + values.resizeCell(idx + 1); + values.add(idx++, encode(i, j, precision)); + } + } + } + return idx; + } + + protected int setValuesByRasterization(String hash, CellValues values, int valuesIndex, int targetPrecision, + MultiGeoValues.GeoValue geoValue) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + GeoRelation relation = relateTile(geoValue, hashes[i]); + if (relation == GeoRelation.QUERY_CROSSES) { + if (hashes[i].length() == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + valuesIndex = + setValuesByRasterization(hashes[i], values, valuesIndex, targetPrecision, geoValue); + } + } else if (relation == GeoRelation.QUERY_INSIDE) { + if (hashes[i].length() == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + values.resizeCell(valuesIndex + (int) Math.pow(32, targetPrecision - hash.length()) + 1); + valuesIndex = setValuesForFullyContainedTile(hashes[i],values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } + + protected int setValuesForFullyContainedTile(String hash, CellValues values, + int valuesIndex, int targetPrecision) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + if (hashes[i].length() == targetPrecision) { + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + valuesIndex = setValuesForFullyContainedTile(hashes[i], values, valuesIndex, targetPrecision); + } + } + return valuesIndex; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoPointCellValues.java similarity index 59% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoPointCellValues.java index e64061eae5ae5..57e7311eee38c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/UnboundedCellValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoPointCellValues.java @@ -18,22 +18,19 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.GeoBoundingBox; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; /** - * Class representing {@link CellValues} that are unbounded by any - * {@link GeoBoundingBox}. + * Class representing geo_point {@link CellValues} */ -class UnboundedCellValues extends CellValues { +class GeoPointCellValues extends CellValues { - UnboundedCellValues(MultiGeoPointValues geoValues, int precision, CellIdSource.GeoPointLongEncoder encoder) { - super(geoValues, precision, encoder); + protected GeoPointCellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { + super(geoValues, precision, tiler); } @Override - int advanceValue(org.elasticsearch.common.geo.GeoPoint target, int valuesIdx) { - values[valuesIdx] = encoder.encode(target.getLon(), target.getLat(), precision); - return valuesIdx + 1; + int advanceValue(MultiGeoValues.GeoValue target, int valuesIdx) { + return tiler.advancePointValue(values, target.lon(), target.lat(), precision, valuesIdx); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoShapeCellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoShapeCellValues.java new file mode 100644 index 0000000000000..22ed207a41338 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoShapeCellValues.java @@ -0,0 +1,35 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.index.fielddata.MultiGeoValues; + +/** Sorted numeric doc values for geo shapes */ +class GeoShapeCellValues extends CellValues { + + protected GeoShapeCellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { + super(geoValues, precision, tiler); + } + + @Override + int advanceValue(MultiGeoValues.GeoValue target, int valuesIdx) { + // TODO(talevy): determine reasonable circuit-breaker here + return tiler.setValues(this, target, precision); + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index 595c6cab6e718..6f3eea16ebfe8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -59,10 +59,10 @@ public GeoGridAggregationBuilder precision(int precision) { } @Override - protected ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, - GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, Map metaData ) throws IOException { + protected ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { return new GeoTileGridAggregatorFactory(name, config, precision, requiredSize, shardSize, geoBoundingBox, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index 0f59c9a71ea40..680639e49a70d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -37,14 +37,14 @@ import java.util.List; import java.util.Map; -public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory { +public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory { private final int precision; private final int requiredSize; private final int shardSize; private final GeoBoundingBox geoBoundingBox; - GeoTileGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, + GeoTileGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, GeoBoundingBox geoBoundingBox, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { @@ -71,7 +71,7 @@ public InternalAggregation buildEmptyAggregation() { } @Override - protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(final ValuesSource.Geo valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, @@ -80,7 +80,13 @@ protected Aggregator doCreateInternal(final ValuesSource.GeoPoint valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, geoBoundingBox, GeoTileUtils::longEncode); + final GeoGridTiler tiler; + if (geoBoundingBox.isUnbounded()) { + tiler = new GeoTileGridTiler(); + } else { + tiler = new BoundedGeoTileGridTiler(geoBoundingBox); + } + CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, tiler); return new GeoTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, searchContext, parent, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTiler.java new file mode 100644 index 0000000000000..869c14c24e252 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTiler.java @@ -0,0 +1,166 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +public class GeoTileGridTiler implements GeoGridTiler { + + @Override + public long encode(double x, double y, int precision) { + return GeoTileUtils.longEncode(x, y, precision); + } + + public int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + values[valuesIdx] = encode(x, y, precision); + return valuesIdx + 1; + } + + /** + * Sets the values of the long[] underlying {@link CellValues}. + * + * If the shape resides between GeoTileUtils.NORMALIZED_LATITUDE_MASK and 90 or + * between GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK and -90 degree latitudes, then + * the shape is not accounted for since geo-tiles are only defined within those bounds. + * + * @param values the bucket values + * @param geoValue the input shape + * @param precision the tile zoom-level + * + * @return the number of tiles set by the shape + */ + @Override + public int setValues(CellValues values, MultiGeoValues.GeoValue geoValue, int precision) { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + assert bounds.minX() <= bounds.maxX(); + + if (precision == 0) { + values.resizeCell(1); + values.add(0, GeoTileUtils.longEncodeTiles(0, 0, 0)); + return 1; + } + + // geo tiles are not defined at the extreme latitudes due to them + // tiling the world as a square. + if ((bounds.top > GeoTileUtils.NORMALIZED_LATITUDE_MASK && bounds.bottom > GeoTileUtils.NORMALIZED_LATITUDE_MASK) + || (bounds.top < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK + && bounds.bottom < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK)) { + return 0; + } + + final double tiles = 1 << precision; + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + int count = (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1); + if (count == 1) { + return setValue(values, geoValue, minXTile, minYTile, precision); + } else if (count <= precision) { + return setValuesByBruteForceScan(values, geoValue, precision, minXTile, minYTile, maxXTile, maxYTile); + } else { + return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue); + } + } + + protected GeoRelation relateTile(MultiGeoValues.GeoValue geoValue, int xTile, int yTile, int precision) { + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + return geoValue.relate(rectangle); + } + + /** + * Sets a singular doc-value for the {@link MultiGeoValues.GeoValue}. To be overriden by {@link BoundedGeoTileGridTiler} + * to account for {@link org.elasticsearch.common.geo.GeoBoundingBox} conditions + */ + protected int setValue(CellValues docValues, MultiGeoValues.GeoValue geoValue, int xTile, int yTile, int precision) { + docValues.resizeCell(1); + docValues.add(0, GeoTileUtils.longEncodeTiles(precision, xTile, yTile)); + return 1; + } + + /** + * + * @param values the bucket values as longs + * @param geoValue the shape value + * @param precision the target precision to split the shape up into + * @return the number of buckets the geoValue is found in + */ + protected int setValuesByBruteForceScan(CellValues values, MultiGeoValues.GeoValue geoValue, + int precision, int minXTile, int minYTile, int maxXTile, int maxYTile) { + int idx = 0; + for (int i = minXTile; i <= maxXTile; i++) { + for (int j = minYTile; j <= maxYTile; j++) { + GeoRelation relation = relateTile(geoValue, i, j, precision); + if (relation != GeoRelation.QUERY_DISJOINT) { + values.resizeCell(idx + 1); + values.add(idx++, GeoTileUtils.longEncodeTiles(precision, i, j)); + } + } + } + return idx; + } + + protected int setValuesByRasterization(int xTile, int yTile, int zTile, CellValues values, int valuesIndex, + int targetPrecision, MultiGeoValues.GeoValue geoValue) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + GeoRelation relation = relateTile(geoValue, nextX, nextY, zTile); + if (GeoRelation.QUERY_INSIDE == relation) { + if (zTile == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + values.resizeCell(valuesIndex + 1 << ( 2 * (targetPrecision - zTile)) + 1); + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } else if (GeoRelation.QUERY_CROSSES == relation) { + if (zTile == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, targetPrecision, geoValue); + } + } + } + } + return valuesIndex; + } + + protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, CellValues values, int valuesIndex, + int targetPrecision) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + if (zTile == targetPrecision) { + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 03f821296f2a6..d872a8c7ce640 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -18,12 +18,15 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.util.SloppyMath; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.util.ESSloppyMath; import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.geometry.Rectangle; import java.io.IOException; import java.util.Locale; @@ -43,6 +46,8 @@ public final class GeoTileUtils { private GeoTileUtils() {} + private static final double PI_DIV_2 = Math.PI / 2; + /** * Largest number of tiles (precision) to use. * This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself (0-31) @@ -53,6 +58,18 @@ private GeoTileUtils() {} */ public static final int MAX_ZOOM = 29; + /** + * The geo-tile map is clipped at 85.05112878 to 90 and -85.05112878 to -90 + */ + public static final double LATITUDE_MASK = 85.0511287798066; + + /** + * Since shapes are encoded, their boundaries are to be compared to against the encoded/decoded values of LATITUDE_MASK + */ + static final double NORMALIZED_LATITUDE_MASK = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(LATITUDE_MASK)); + static final double NORMALIZED_NEGATIVE_LATITUDE_MASK = + GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(-LATITUDE_MASK)); + /** * Bit position of the zoom value within hash - zoom is stored in the most significant 6 bits of a long number. */ @@ -63,6 +80,7 @@ private GeoTileUtils() {} */ private static final long X_Y_VALUE_MASK = (1L << MAX_ZOOM) - 1; + /** * Parse an integer precision (zoom level). The {@link ValueType#INT} allows it to be a number or a string. * @@ -90,37 +108,65 @@ public static int checkPrecisionRange(int precision) { } /** - * Encode lon/lat to the geotile based long format. - * The resulting hash contains interleaved tile X and Y coordinates. - * The precision itself is also encoded as a few high bits. + * Calculates the x-coordinate in the tile grid for the specified longitude given + * the number of tile columns for a pre-determined zoom-level. + * + * @param longitude the longitude to use when determining the tile x-coordinate + * @param tiles the number of tiles per row for a pre-determined zoom-level */ - public static long longEncode(double longitude, double latitude, int precision) { - // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java - - // Number of tiles for the current zoom level along the X and Y axis - final long tiles = 1 << checkPrecisionRange(precision); - - long xTile = (long) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); + static int getXTile(double longitude, long tiles) { + // normalizeLon treats this as 180, which is not friendly for tile mapping + if (longitude == -180) { + return 0; + } - double latSin = Math.sin(Math.toRadians(normalizeLat(latitude))); - long yTile = (long) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); + int xTile = (int) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); // Edge values may generate invalid values, and need to be clipped. // For example, polar regions (above/below lat 85.05112878) get normalized. if (xTile < 0) { - xTile = 0; + return 0; } if (xTile >= tiles) { - xTile = tiles - 1; + return (int) tiles - 1; } + + return xTile; + } + + /** + * Calculates the y-coordinate in the tile grid for the specified longitude given + * the number of tile rows for pre-determined zoom-level. + * + * @param latitude the latitude to use when determining the tile y-coordinate + * @param tiles the number of tiles per column for a pre-determined zoom-level + */ + static int getYTile(double latitude, long tiles) { + double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(normalizeLat(latitude))); + int yTile = (int) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); + if (yTile < 0) { yTile = 0; } if (yTile >= tiles) { - yTile = tiles - 1; + return (int) tiles - 1; } - return longEncode((long) precision, xTile, yTile); + return yTile; + } + + /** + * Encode lon/lat to the geotile based long format. + * The resulting hash contains interleaved tile X and Y coordinates. + * The precision itself is also encoded as a few high bits. + */ + public static long longEncode(double longitude, double latitude, int precision) { + // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java + // Number of tiles for the current zoom level along the X and Y axis + final long tiles = 1 << checkPrecisionRange(precision); + long xTile = getXTile(longitude, tiles); + long yTile = getYTile(latitude, tiles); + return longEncodeTiles(precision, xTile, yTile); } /** @@ -131,7 +177,14 @@ public static long longEncode(double longitude, double latitude, int precision) */ public static long longEncode(String hashAsString) { int[] parsed = parseHash(hashAsString); - return longEncode((long)parsed[0], (long)parsed[1], (long)parsed[2]); + return longEncode((long) parsed[0], (long) parsed[1], (long) parsed[2]); + } + + static long longEncodeTiles(int precision, long xTile, long yTile) { + // Zoom value is placed in front of all the bits used for the geotile + // e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th), + // leaving 5 bits unused for zoom. See MAX_ZOOM comment above. + return ((long) precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile; } /** @@ -193,6 +246,23 @@ static GeoPoint keyToGeoPoint(String hashAsString) { return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]); } + static Rectangle toBoundingBox(long hash) { + int[] hashAsInts = parseHash(hash); + return toBoundingBox(hashAsInts[1], hashAsInts[2], hashAsInts[0]); + } + + static Rectangle toBoundingBox(int xTile, int yTile, int precision) { + final double tiles = validateZXY(precision, xTile, yTile); + final double minN = Math.PI - (2.0 * Math.PI * (yTile + 1)) / tiles; + final double maxN = Math.PI - (2.0 * Math.PI * (yTile)) / tiles; + final double minY = Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(minN))); + final double minX = ((xTile) / tiles * 360.0) - 180; + final double maxY = Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(maxN))); + final double maxX = ((xTile + 1) / tiles * 360.0) - 180; + + return new Rectangle(minX, maxX, maxY, minY); + } + /** * Validates Zoom, X, and Y values, and returns the total number of allowed tiles along the x/y axis. */ diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java index 92d8bbc15a754..c3b281a78b545 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java @@ -60,7 +60,7 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde private static final ObjectParser PARSER; static { PARSER = new ObjectParser<>(GeoDistanceAggregationBuilder.NAME); - ValuesSourceParserHelper.declareGeoFields(PARSER, true, false); + ValuesSourceParserHelper.declareGeoPointFields(PARSER, true, false); PARSER.declareBoolean(GeoDistanceAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java index 711297762b8d4..46bdec14a9d53 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.query.QueryShardContext; @@ -115,7 +115,7 @@ public SortedNumericDocValues longValues(LeafReaderContext ctx) { @Override public SortedNumericDoubleValues doubleValues(LeafReaderContext ctx) { - final MultiGeoPointValues geoValues = source.geoPointValues(ctx); + final MultiGeoValues geoValues = source.geoValues(ctx); return GeoUtils.distanceValues(distanceType, units, geoValues, origin); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java index 8b9f89fd54209..e0140bf94cf54 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java @@ -39,7 +39,7 @@ import java.util.Map; import java.util.Objects; -public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder { +public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "geo_bounds"; private static final ObjectParser PARSER; @@ -56,7 +56,7 @@ public static AggregationBuilder parse(String aggregationName, XContentParser pa private boolean wrapLongitude = true; public GeoBoundsAggregationBuilder(String name) { - super(name, CoreValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(name, CoreValuesSourceType.GEO, ValueType.GEO); } protected GeoBoundsAggregationBuilder(GeoBoundsAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -73,7 +73,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected GeoBoundsAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new GeoBoundsAggregatorFactory(name, config, wrapLongitude, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java index e6d591482be2b..39aac155828b3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java @@ -21,11 +21,10 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.DoubleArray; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.LeafBucketCollector; @@ -42,7 +41,7 @@ final class GeoBoundsAggregator extends MetricsAggregator { static final ParseField WRAP_LONGITUDE_FIELD = new ParseField("wrap_longitude"); - private final ValuesSource.GeoPoint valuesSource; + private final ValuesSource.Geo valuesSource; private final boolean wrapLongitude; DoubleArray tops; DoubleArray bottoms; @@ -52,7 +51,7 @@ final class GeoBoundsAggregator extends MetricsAggregator { DoubleArray negRights; GeoBoundsAggregator(String name, SearchContext aggregationContext, Aggregator parent, - ValuesSource.GeoPoint valuesSource, boolean wrapLongitude, List pipelineAggregators, + ValuesSource.Geo valuesSource, boolean wrapLongitude, List pipelineAggregators, Map metaData) throws IOException { super(name, aggregationContext, parent, pipelineAggregators, metaData); this.valuesSource = valuesSource; @@ -81,7 +80,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - final MultiGeoPointValues values = valuesSource.geoPointValues(ctx); + final MultiGeoValues values = valuesSource.geoValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -105,31 +104,14 @@ public void collect(int doc, long bucket) throws IOException { final int valuesCount = values.docValueCount(); for (int i = 0; i < valuesCount; ++i) { - GeoPoint value = values.nextValue(); - double top = tops.get(bucket); - if (value.lat() > top) { - top = value.lat(); - } - double bottom = bottoms.get(bucket); - if (value.lat() < bottom) { - bottom = value.lat(); - } - double posLeft = posLefts.get(bucket); - if (value.lon() >= 0 && value.lon() < posLeft) { - posLeft = value.lon(); - } - double posRight = posRights.get(bucket); - if (value.lon() >= 0 && value.lon() > posRight) { - posRight = value.lon(); - } - double negLeft = negLefts.get(bucket); - if (value.lon() < 0 && value.lon() < negLeft) { - negLeft = value.lon(); - } - double negRight = negRights.get(bucket); - if (value.lon() < 0 && value.lon() > negRight) { - negRight = value.lon(); - } + MultiGeoValues.GeoValue value = values.nextValue(); + MultiGeoValues.BoundingBox bounds = value.boundingBox(); + double top = Math.max(tops.get(bucket), bounds.top); + double bottom = Math.min(bottoms.get(bucket), bounds.bottom); + double posLeft = Math.min(posLefts.get(bucket), bounds.posLeft); + double posRight = Math.max(posRights.get(bucket), bounds.posRight); + double negLeft = Math.min(negLefts.get(bucket), bounds.negLeft); + double negRight = Math.max(negRights.get(bucket), bounds.negRight); tops.set(bucket, top); bottoms.set(bucket, bottom); posLefts.set(bucket, posLeft); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java index 462aff381d3d8..ffc0643f583cb 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java @@ -33,12 +33,12 @@ import java.util.List; import java.util.Map; -class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory { +class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory { private final boolean wrapLongitude; GeoBoundsAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, boolean wrapLongitude, QueryShardContext queryShardContext, AggregatorFactory parent, @@ -57,7 +57,7 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(ValuesSource.Geo valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java index daf357186f36c..a6200097797a0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java @@ -39,7 +39,7 @@ import java.util.Map; public class GeoCentroidAggregationBuilder - extends ValuesSourceAggregationBuilder.LeafOnly { + extends ValuesSourceAggregationBuilder.LeafOnly { public static final String NAME = "geo_centroid"; private static final ObjectParser PARSER; @@ -78,7 +78,7 @@ protected void innerWriteTo(StreamOutput out) { } @Override - protected GeoCentroidAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, + protected GeoCentroidAggregatorFactory innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new GeoCentroidAggregatorFactory(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java index bf318896e55e0..72e4146dd6b0d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java @@ -20,12 +20,14 @@ package org.elasticsearch.search.aggregations.metrics; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.geo.DimensionalShapeType; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ByteArray; import org.elasticsearch.common.util.DoubleArray; import org.elasticsearch.common.util.LongArray; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.LeafBucketCollector; @@ -42,13 +44,14 @@ * A geo metric aggregator that computes a geo-centroid from a {@code geo_point} type field */ final class GeoCentroidAggregator extends MetricsAggregator { - private final ValuesSource.GeoPoint valuesSource; - private DoubleArray lonSum, lonCompensations, latSum, latCompensations; + private final ValuesSource.Geo valuesSource; + private DoubleArray lonSum, lonCompensations, latSum, latCompensations, weightSum, weightCompensations; private LongArray counts; + private ByteArray dimensionalShapeTypes; GeoCentroidAggregator(String name, SearchContext context, Aggregator parent, - ValuesSource.GeoPoint valuesSource, List pipelineAggregators, - Map metaData) throws IOException { + ValuesSource.Geo valuesSource, List pipelineAggregators, + Map metaData) throws IOException { super(name, context, parent, pipelineAggregators, metaData); this.valuesSource = valuesSource; if (valuesSource != null) { @@ -57,7 +60,10 @@ final class GeoCentroidAggregator extends MetricsAggregator { lonCompensations = bigArrays.newDoubleArray(1, true); latSum = bigArrays.newDoubleArray(1, true); latCompensations = bigArrays.newDoubleArray(1, true); + weightSum = bigArrays.newDoubleArray(1, true); + weightCompensations = bigArrays.newDoubleArray(1, true); counts = bigArrays.newLongArray(1, true); + dimensionalShapeTypes = bigArrays.newByteArray(1, true); } } @@ -67,18 +73,22 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - final MultiGeoPointValues values = valuesSource.geoPointValues(ctx); + final MultiGeoValues values = valuesSource.geoValues(ctx); final CompensatedSum compensatedSumLat = new CompensatedSum(0, 0); final CompensatedSum compensatedSumLon = new CompensatedSum(0, 0); + final CompensatedSum compensatedSumWeight = new CompensatedSum(0, 0); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { latSum = bigArrays.grow(latSum, bucket + 1); lonSum = bigArrays.grow(lonSum, bucket + 1); + weightSum = bigArrays.grow(weightSum, bucket + 1); lonCompensations = bigArrays.grow(lonCompensations, bucket + 1); latCompensations = bigArrays.grow(latCompensations, bucket + 1); + weightCompensations = bigArrays.grow(weightCompensations, bucket + 1); counts = bigArrays.grow(counts, bucket + 1); + dimensionalShapeTypes = bigArrays.grow(dimensionalShapeTypes, bucket + 1); if (values.advanceExact(doc)) { final int valueCount = values.docValueCount(); @@ -86,26 +96,47 @@ public void collect(int doc, long bucket) throws IOException { counts.increment(bucket, valueCount); // Compute the sum of double values with Kahan summation algorithm which is more // accurate than naive summation. + DimensionalShapeType shapeType = DimensionalShapeType.fromOrdinalByte(dimensionalShapeTypes.get(bucket)); double sumLat = latSum.get(bucket); double compensationLat = latCompensations.get(bucket); double sumLon = lonSum.get(bucket); double compensationLon = lonCompensations.get(bucket); + double sumWeight = weightSum.get(bucket); + double compensatedWeight = weightCompensations.get(bucket); compensatedSumLat.reset(sumLat, compensationLat); compensatedSumLon.reset(sumLon, compensationLon); + compensatedSumWeight.reset(sumWeight, compensatedWeight); // update the sum for (int i = 0; i < valueCount; ++i) { - GeoPoint value = values.nextValue(); - //latitude - compensatedSumLat.add(value.getLat()); - //longitude - compensatedSumLon.add(value.getLon()); + MultiGeoValues.GeoValue value = values.nextValue(); + int compares = shapeType.compareTo(value.dimensionalShapeType()); + if (compares < 0) { + double coordinateWeight = value.weight(); + compensatedSumLat.reset(coordinateWeight * value.lat(), 0.0); + compensatedSumLon.reset(coordinateWeight * value.lon(), 0.0); + compensatedSumWeight.reset(coordinateWeight, 0.0); + dimensionalShapeTypes.set(bucket, (byte) value.dimensionalShapeType().ordinal()); + } else if (compares == 0) { + double coordinateWeight = value.weight(); + // weighted latitude + compensatedSumLat.add(coordinateWeight * value.lat()); + // weighted longitude + compensatedSumLon.add(coordinateWeight * value.lon()); + // weight + compensatedSumWeight.add(coordinateWeight); + } + // else (compares > 0) + // do not modify centroid calculation since shape is of lower dimension than the running dimension + } lonSum.set(bucket, compensatedSumLon.value()); lonCompensations.set(bucket, compensatedSumLon.delta()); latSum.set(bucket, compensatedSumLat.value()); latCompensations.set(bucket, compensatedSumLat.delta()); + weightSum.set(bucket, compensatedSumWeight.value()); + weightCompensations.set(bucket, compensatedSumWeight.delta()); } } }; @@ -117,9 +148,10 @@ public InternalAggregation buildAggregation(long bucket) { return buildEmptyAggregation(); } final long bucketCount = counts.get(bucket); - final GeoPoint bucketCentroid = (bucketCount > 0) - ? new GeoPoint(latSum.get(bucket) / bucketCount, lonSum.get(bucket) / bucketCount) - : null; + final double bucketWeight = weightSum.get(bucket); + final GeoPoint bucketCentroid = (bucketWeight > 0) + ? new GeoPoint(latSum.get(bucket) / bucketWeight, lonSum.get(bucket) / bucketWeight) + : null; return new InternalGeoCentroid(name, bucketCentroid , bucketCount, pipelineAggregators(), metaData()); } @@ -130,6 +162,7 @@ public InternalAggregation buildEmptyAggregation() { @Override public void doClose() { - Releasables.close(latSum, latCompensations, lonSum, lonCompensations, counts); + Releasables.close(latSum, latCompensations, lonSum, lonCompensations, counts, weightSum, weightCompensations, + dimensionalShapeTypes); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java index 73200f18d91af..e29b819d83776 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java @@ -33,10 +33,10 @@ import java.util.List; import java.util.Map; -class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory { +class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory { GeoCentroidAggregatorFactory(String name, - ValuesSourceConfig config, + ValuesSourceConfig config, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, @@ -53,7 +53,7 @@ protected Aggregator createUnmapped(SearchContext searchContext, } @Override - protected Aggregator doCreateInternal(ValuesSource.GeoPoint valuesSource, + protected Aggregator doCreateInternal(ValuesSource.Geo valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java index 15585c2f4a3a8..22249d8aa6a94 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java @@ -175,7 +175,8 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th return builder; } - private GeoBoundingBox resolveGeoBoundingBox() { + // used for testing + GeoBoundingBox resolveGeoBoundingBox() { if (Double.isInfinite(top)) { return null; } else if (Double.isInfinite(posLeft)) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index 6df32b4deefa0..47bbcba801b78 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -20,15 +20,18 @@ package org.elasticsearch.search.aggregations.support; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.script.AggregationScript; @@ -164,7 +167,8 @@ public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFa @Override public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { // TODO: also support the structured formats of geo points - final GeoPoint missing = new GeoPoint(rawMissing.toString()); + final GeoPoint missingPoint = new GeoPoint(rawMissing.toString()); + final MultiGeoValues.GeoPointValue missing = new MultiGeoValues.GeoPointValue(missingPoint); return MissingValues.replaceMissing((ValuesSource.GeoPoint) valuesSource, missing); } }, @@ -224,6 +228,75 @@ public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFa public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { throw new IllegalArgumentException("Can't apply missing values on a " + valuesSource.getClass()); } + }, + GEOSHAPE { + @Override + public ValuesSource getEmpty() { + return ValuesSource.GeoShape.EMPTY; + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + throw new AggregationExecutionException("value source of type [" + this.value() + "] is not supported by scripts"); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + if (!(fieldContext.indexFieldData() instanceof IndexGeoShapeFieldData)) { + // TODO: Is this the correct exception type here? + throw new IllegalArgumentException("Expected geo_shape type on field [" + fieldContext.field() + + "], but got [" + fieldContext.fieldType().typeName() + "]"); + } + + return new ValuesSource.GeoShape.Fielddata((IndexGeoShapeFieldData) fieldContext.indexFieldData()); + } + + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + final MultiGeoValues.GeoShapeValue missing = MultiGeoValues.GeoShapeValue.missing(rawMissing.toString()); + return MissingValues.replaceMissing((ValuesSource.GeoShape) valuesSource, missing); + } + }, + GEO { + @Override + public ValuesSource getEmpty() { + return ValuesSource.Geo.EMPTY; + } + + @Override + public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + // TODO (support scripts) + throw new UnsupportedOperationException("CoreValuesSourceType.GEO is still a special case"); + } + + @Override + public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { + boolean isGeoPoint = fieldContext.indexFieldData() instanceof IndexGeoPointFieldData; + boolean isGeoShape = fieldContext.indexFieldData() instanceof IndexGeoShapeFieldData; + if (isGeoPoint == false && isGeoShape == false) { + throw new IllegalArgumentException("Expected geo_point or geo_shape type on field [" + fieldContext.field() + + "], but got [" + fieldContext.fieldType().typeName() + "]"); + } + if (isGeoPoint) { + return new ValuesSource.GeoPoint.Fielddata((IndexGeoPointFieldData) fieldContext.indexFieldData()); + } + return new ValuesSource.GeoShape.Fielddata((IndexGeoShapeFieldData) fieldContext.indexFieldData()); + } + + @Override + public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { + // when missing value is present on aggregations that support both shapes and points, geo_point will be + // assumed first to preserve backwards compatibility with existing behavior. If a value is not a valid geo_point + // then it is parsed as a geo_shape + try { + final MultiGeoValues.GeoPointValue missing = new + MultiGeoValues.GeoPointValue(new GeoPoint(rawMissing.toString())); + return MissingValues.replaceMissing(ValuesSource.GeoPoint.EMPTY, missing); + } catch (ElasticsearchParseException e) { + return MissingValues.replaceMissing(ValuesSource.GeoShape.EMPTY, + MultiGeoValues.GeoShapeValue.missing(rawMissing.toString())); + } + } }; public static ValuesSourceType fromString(String name) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index c61091fd2a12c..86ca9c0a79076 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -23,10 +23,9 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -396,29 +395,8 @@ static LongUnaryOperator getGlobalMapping(SortedSetDocValues values, SortedSetDo } } - public static ValuesSource.GeoPoint replaceMissing(final ValuesSource.GeoPoint valuesSource, final GeoPoint missing) { - return new ValuesSource.GeoPoint() { - - @Override - public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return replaceMissing(valuesSource.bytesValues(context), new BytesRef(missing.toString())); - } - - @Override - public MultiGeoPointValues geoPointValues(LeafReaderContext context) { - final MultiGeoPointValues values = valuesSource.geoPointValues(context); - return replaceMissing(values, missing); - } - - @Override - public String toString() { - return "anon ValuesSource.GeoPoint of [" + super.toString() + "]"; - } - }; - } - - static MultiGeoPointValues replaceMissing(final MultiGeoPointValues values, final GeoPoint missing) { - return new MultiGeoPointValues() { + static MultiGeoValues replaceMissing(final MultiGeoValues values, final MultiGeoValues.GeoValue missing) { + return new MultiGeoValues() { private int count; @@ -440,7 +418,12 @@ public int docValueCount() { } @Override - public GeoPoint nextValue() throws IOException { + public ValuesSourceType valuesSourceType() { + return values.valuesSourceType(); + } + + @Override + public GeoValue nextValue() throws IOException { if (count > 0) { return values.nextValue(); } else { @@ -454,4 +437,38 @@ public String toString() { } }; } + + public static ValuesSource.GeoPoint replaceMissing(final ValuesSource.GeoPoint valuesSource, + final MultiGeoValues.GeoPointValue missing) { + return new ValuesSource.GeoPoint() { + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return replaceMissing(valuesSource.bytesValues(context), new BytesRef(missing.toString())); + } + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + final MultiGeoValues values = valuesSource.geoValues(context); + return replaceMissing(values, missing); + } + }; + } + + public static ValuesSource.GeoShape replaceMissing(final ValuesSource.GeoShape valuesSource, + final MultiGeoValues.GeoShapeValue missing) { + return new ValuesSource.GeoShape() { + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return replaceMissing(valuesSource.bytesValues(context), new BytesRef(missing.toString())); + } + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + final MultiGeoValues values = valuesSource.geoValues(context); + return replaceMissing(values, missing); + } + }; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java index a1e3237288fa9..d6f4ebdbfb40c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java @@ -24,7 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexGeometryFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.plain.BinaryDVIndexFieldData; import org.elasticsearch.index.mapper.DateFieldMapper; @@ -50,7 +52,11 @@ public enum ValueType implements Writeable { NUMERIC((byte) 7, "numeric", "numeric", CoreValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.RAW), GEOPOINT((byte) 8, "geo_point", "geo_point", CoreValuesSourceType.GEOPOINT, IndexGeoPointFieldData.class, DocValueFormat.GEOHASH), BOOLEAN((byte) 9, "boolean", "boolean", CoreValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.BOOLEAN), - RANGE((byte) 10, "range", "range", CoreValuesSourceType.RANGE, BinaryDVIndexFieldData.class, DocValueFormat.RAW); + RANGE((byte) 10, "range", "range", CoreValuesSourceType.RANGE, BinaryDVIndexFieldData.class, DocValueFormat.RAW), + GEOSHAPE((byte) 11, "geo_shape", "geo_shape", CoreValuesSourceType.GEOSHAPE, IndexGeoShapeFieldData.class, DocValueFormat.RAW), + // GEO is an abstract ValueType that represents either GEOPOINT or GEOSHAPE in aggregations. It is never directly + // associated with concrete field data + GEO((byte) 12, "geo", "geo", CoreValuesSourceType.GEO, IndexGeometryFieldData.class, DocValueFormat.RAW); final String description; final ValuesSourceType valuesSourceType; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 3f77744b5a56e..4a762b86c19ff 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -37,9 +37,10 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexHistogramFieldData; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingBinaryDocValues; @@ -131,10 +132,10 @@ public DocValueBits docsWithValue(LeafReaderContext context) throws IOException } public abstract SortedSetDocValues ordinalsValues(LeafReaderContext context) - throws IOException; + throws IOException; public abstract SortedSetDocValues globalOrdinalsValues(LeafReaderContext context) - throws IOException; + throws IOException; /** * Whether this values source is able to provide a mapping between global and segment ordinals, @@ -147,7 +148,7 @@ public boolean supportsGlobalOrdinalsMapping() { /** Returns a mapping from segment ordinals to global ordinals. */ public abstract LongUnaryOperator globalOrdinalsMapping(LeafReaderContext context) - throws IOException; + throws IOException; public long globalMaxOrd(IndexSearcher indexSearcher) throws IOException { IndexReader indexReader = indexSearcher.getIndexReader(); @@ -523,13 +524,82 @@ public boolean needsScores() { } - public abstract static class GeoPoint extends ValuesSource { + // No need to implement ReaderContextAware here, the delegate already takes care of updating data structures + public static class WithScript extends Bytes { - public static final GeoPoint EMPTY = new GeoPoint() { + private final ValuesSource delegate; + private final AggregationScript.LeafFactory script; + + public WithScript(ValuesSource delegate, AggregationScript.LeafFactory script) { + this.delegate = delegate; + this.script = script; + } + + @Override + public boolean needsScores() { + return script.needs_score(); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return new BytesValues(delegate.bytesValues(context), script.newInstance(context)); + } + + static class BytesValues extends SortingBinaryDocValues implements ScorerAware { + + private final SortedBinaryDocValues bytesValues; + private final AggregationScript script; + + BytesValues(SortedBinaryDocValues bytesValues, AggregationScript script) { + this.bytesValues = bytesValues; + this.script = script; + } @Override - public MultiGeoPointValues geoPointValues(LeafReaderContext context) { - return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoPoints(); + public void setScorer(Scorable scorer) { + script.setScorer(scorer); + } + + @Override + public boolean advanceExact(int doc) throws IOException { + if (bytesValues.advanceExact(doc)) { + count = bytesValues.docValueCount(); + grow(); + script.setDocument(doc); + for (int i = 0; i < count; ++i) { + final BytesRef value = bytesValues.nextValue(); + script.setNextAggregationValue(value.utf8ToString()); + Object run = script.execute(); + CollectionUtils.ensureNoSelfReferences(run, "ValuesSource.BytesValues script"); + values[i].copyChars(run.toString()); + } + sort(); + return true; + } else { + count = 0; + grow(); + return false; + } + } + } + } + + /** + * This class represents abstract geo fields that can either be geo_point or geo_shape + */ + public abstract static class Geo extends ValuesSource { + public abstract MultiGeoValues geoValues(LeafReaderContext context); + + @Override + public DocValueBits docsWithValue(LeafReaderContext context) { + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoValues(context)); + } + + public static final Geo EMPTY = new Geo() { + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoValues(); } @Override @@ -538,14 +608,24 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc } }; + } - @Override - public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { - final MultiGeoPointValues geoPoints = geoPointValues(context); - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoPoints); - } + public abstract static class GeoPoint extends Geo { + + public static final GeoPoint EMPTY = new GeoPoint() { + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + return Geo.EMPTY.geoValues(context); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return Geo.EMPTY.bytesValues(context); + } + + }; - public abstract MultiGeoPointValues geoPointValues(LeafReaderContext context); public static class Fielddata extends GeoPoint { @@ -560,12 +640,47 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) { return indexFieldData.load(context).getBytesValues(); } - public org.elasticsearch.index.fielddata.MultiGeoPointValues geoPointValues(LeafReaderContext context) { - return indexFieldData.load(context).getGeoPointValues(); + public MultiGeoValues geoValues(LeafReaderContext context) { + return indexFieldData.load(context).getGeoValues(); } } } - + + public abstract static class GeoShape extends Geo { + + public static final GeoShape EMPTY = new GeoShape() { + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + return Geo.EMPTY.geoValues(context); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return Geo.EMPTY.bytesValues(context); + } + + }; + + public static class Fielddata extends GeoShape { + + protected final IndexGeoShapeFieldData indexFieldData; + + public Fielddata(IndexGeoShapeFieldData indexFieldData) { + this.indexFieldData = indexFieldData; + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) { + return indexFieldData.load(context).getBytesValues(); + } + + public MultiGeoValues geoValues(LeafReaderContext context) { + return indexFieldData.load(context).getGeoValues(); + } + } + } + public abstract static class Histogram extends ValuesSource { public abstract HistogramValues getHistogramValues(LeafReaderContext context) throws IOException; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index ba599827b88af..91aa4b41880bf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; import org.elasticsearch.index.fielddata.IndexHistogramFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.DateFieldMapper; @@ -115,6 +116,8 @@ public static ValuesSourceConfig resolve( config = new ValuesSourceConfig<>(CoreValuesSourceType.GEOPOINT); } else if (fieldType instanceof RangeFieldMapper.RangeFieldType) { config = new ValuesSourceConfig<>(CoreValuesSourceType.RANGE); + } else if (indexFieldData instanceof IndexGeoShapeFieldData) { + config = new ValuesSourceConfig<>(CoreValuesSourceType.GEOSHAPE); } else if (indexFieldData instanceof IndexHistogramFieldData) { config = new ValuesSourceConfig<>(CoreValuesSourceType.HISTOGRAM); } else { @@ -250,6 +253,7 @@ public VS toValuesSource(QueryShardContext context) { /** Get a value source given its configuration. A return value of null indicates that * no value source could be built. */ @Nullable + @SuppressWarnings("unchecked") public VS toValuesSource(QueryShardContext context, Function resolveMissingAny) { if (!valid()) { throw new IllegalStateException( diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java index 567862ca92e3e..f2dd081cdf7b7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java @@ -57,12 +57,18 @@ public static void declareBytesFields( declareFields(objectParser, scriptable, formattable, false, ValueType.STRING); } - public static void declareGeoFields( + public static void declareGeoPointFields( AbstractObjectParser, T> objectParser, boolean scriptable, boolean formattable) { declareFields(objectParser, scriptable, formattable, false, ValueType.GEOPOINT); } + public static void declareGeoFields( + AbstractObjectParser, T> objectParser, + boolean scriptable, boolean formattable) { + declareFields(objectParser, scriptable, formattable, false, ValueType.GEO); + } + private static void declareFields( AbstractObjectParser, T> objectParser, boolean scriptable, boolean formattable, boolean timezoneAware, ValueType targetValueType) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index dffb8dab9016f..8e6aef409349e 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -44,7 +44,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointDVIndexFieldData.LatLonPointDVIndexFieldData; @@ -594,7 +594,7 @@ public SortField.Type reducedType() { } private NumericDoubleValues getNumericDoubleValues(LeafReaderContext context) throws IOException { - final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues(); + final MultiGeoValues geoPointValues = geoIndexFieldData.load(context).getGeoValues(); final SortedNumericDoubleValues distanceValues = GeoUtils.distanceValues(geoDistance, unit, geoPointValues, localPoints); if (nested == null) { return FieldData.replaceMissing(sortMode.select(distanceValues), Double.POSITIVE_INFINITY); diff --git a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java new file mode 100644 index 0000000000000..9cf05e08b7d91 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java @@ -0,0 +1,408 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.common.geo.DimensionalShapeType.LINE; +import static org.elasticsearch.common.geo.DimensionalShapeType.POINT; +import static org.elasticsearch.common.geo.DimensionalShapeType.POLYGON; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + +public class CentroidCalculatorTests extends ESTestCase { + private static final double DELTA = 0.000000001; + + public void testPoint() { + Point point = GeometryTestUtils.randomPoint(false); + CentroidCalculator calculator = new CentroidCalculator(point); + assertThat(calculator.getX(), equalTo(GeoUtils.normalizeLon(point.getX()))); + assertThat(calculator.getY(), equalTo(GeoUtils.normalizeLat(point.getY()))); + assertThat(calculator.sumWeight(), equalTo(1.0)); + assertThat(calculator.getDimensionalShapeType(), equalTo(POINT)); + } + + public void testPolygonWithSmallTrianglesOfZeroWeight() throws Exception { + Geometry geometry = new WellKnownText(false, new GeographyValidator(true)) + .fromWKT("POLYGON((-4.385064 55.2259599,-4.385056 55.2259224,-4.3850466 55.2258994,-4.3849755 55.2258574," + + "-4.3849339 55.2258589,-4.3847033 55.2258742,-4.3846805 55.2258818,-4.3846282 55.2259132,-4.3846215 55.2259247," + + "-4.3846121 55.2259683,-4.3846147 55.2259798,-4.3846369 55.2260157,-4.3846472 55.2260241," + + "-4.3846697 55.2260409,-4.3846952 55.2260562,-4.384765 55.22608,-4.3848199 55.2260861,-4.3848481 55.2260845," + + "-4.3849245 55.2260761,-4.3849393 55.22607,-4.3849996 55.2260432,-4.3850131 55.2260364,-4.3850426 55.2259989," + + "-4.385064 55.2259599),(-4.3850104 55.2259583,-4.385005 55.2259752,-4.384997 55.2259892,-4.3849339 55.2259981," + + "-4.3849272 55.2259308,-4.3850016 55.2259262,-4.385005 55.2259377,-4.3850104 55.2259583)," + + "(-4.3849996 55.2259193,-4.3847502 55.2259331,-4.3847548 55.2258921,-4.3848012 55.2258895," + + "-4.3849219 55.2258811,-4.3849514 55.2258818,-4.3849728 55.2258933,-4.3849996 55.2259193)," + + "(-4.3849917 55.2259984,-4.3849849 55.2260103,-4.3849771 55.2260192,-4.3849701 55.2260019,-4.3849917 55.2259984)," + + "(-4.3846608 55.2259374,-4.384663 55.2259316,-4.3846711 55.2259201,-4.3846992 55.225904," + + "-4.384718 55.2258941,-4.3847434 55.2258927,-4.3847314 55.2259407,-4.3849098 55.2259316,-4.3849098 55.2259492," + + "-4.3848843 55.2259515,-4.3849017 55.2260119,-4.3849567 55.226005,-4.3849701 55.2260272,-4.3849299 55.2260486," + + "-4.3849192 55.2260295,-4.384883 55.2260188,-4.3848776 55.2260119,-4.3848441 55.2260149,-4.3848441 55.2260226," + + "-4.3847864 55.2260241,-4.384722 55.2259652,-4.3847053 55.2259706,-4.384683 55.225954,-4.3846608 55.2259374)," + + "(-4.3846541 55.2259549,-4.384698 55.2259883,-4.3847173 55.2259828,-4.3847743 55.2260333,-4.3847891 55.2260356," + + "-4.3848146 55.226031,-4.3848199 55.2260409,-4.3848387 55.2260417,-4.3848494 55.2260593,-4.3848092 55.2260616," + + "-4.3847623 55.2260539,-4.3847341 55.2260432,-4.3847046 55.2260279,-4.3846738 55.2260062,-4.3846496 55.2259844," + + "-4.3846429 55.2259737,-4.3846523 55.2259714,-4.384651 55.2259629,-4.3846541 55.2259549)," + + "(-4.3846608 55.2259374,-4.3846559 55.2259502,-4.3846541 55.2259549,-4.3846608 55.2259374))"); + CentroidCalculator calculator = new CentroidCalculator(geometry); + assertThat(calculator.getX(), closeTo( -4.3848, 1e-4)); + assertThat(calculator.getY(), closeTo(55.22595, 1e-4)); + assertThat(calculator.sumWeight(), closeTo(0, 1e-5)); + assertThat(calculator.getDimensionalShapeType(), equalTo(POLYGON)); + } + + + public void testLine() { + double[] y = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + double[] x = new double[] { 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 }; + double[] yRunningAvg = new double[] { 1, 1.5, 2.0, 2.5, 3, 3.5, 4, 4.5, 5, 5.5 }; + double[] xRunningAvg = new double[] { 10, 15, 20, 25, 30, 35, 40, 45, 50, 55 }; + + Point point = new Point(x[0], y[0]); + CentroidCalculator calculator = new CentroidCalculator(point); + assertThat(calculator.getX(), equalTo(xRunningAvg[0])); + assertThat(calculator.getY(), equalTo(yRunningAvg[0])); + for (int i = 1; i < 10; i++) { + double[] subX = new double[i + 1]; + double[] subY = new double[i + 1]; + System.arraycopy(x, 0, subX, 0, i + 1); + System.arraycopy(y, 0, subY, 0, i + 1); + Geometry geometry = new Line(subX, subY); + calculator = new CentroidCalculator(geometry); + assertEquals(xRunningAvg[i], calculator.getX(), DELTA); + assertEquals(yRunningAvg[i], calculator.getY(), DELTA); + } + CentroidCalculator otherCalculator = new CentroidCalculator(new Point(0, 0)); + calculator.addFrom(otherCalculator); + assertEquals(55.0, calculator.getX(), DELTA); + assertEquals(5.5, calculator.getY(), DELTA); + } + + public void testMultiLine() { + MultiLine multiLine = GeometryTestUtils.randomMultiLine(false); + double sumLineX = 0; + double sumLineY = 0; + double sumLineWeight = 0; + for (Line line : multiLine) { + CentroidCalculator calculator = new CentroidCalculator(line); + sumLineX += calculator.compSumX.value(); + sumLineY += calculator.compSumY.value(); + sumLineWeight += calculator.compSumWeight.value(); + } + CentroidCalculator calculator = new CentroidCalculator(multiLine); + + assertEquals(sumLineX / sumLineWeight, calculator.getX(), DELTA); + assertEquals(sumLineY / sumLineWeight, calculator.getY(), DELTA); + assertEquals(sumLineWeight, calculator.sumWeight(), DELTA); + assertThat(calculator.getDimensionalShapeType(), equalTo(LINE)); + } + + public void testMultiPoint() { + MultiPoint multiPoint = GeometryTestUtils.randomMultiPoint(false); + double sumPointX = 0; + double sumPointY = 0; + double sumPointWeight = 0; + for (Point point : multiPoint) { + sumPointX += point.getX(); + sumPointY += point.getY(); + sumPointWeight += 1; + } + + CentroidCalculator calculator = new CentroidCalculator(multiPoint); + assertEquals(sumPointX / sumPointWeight, calculator.getX(), DELTA); + assertEquals(sumPointY / sumPointWeight, calculator.getY(), DELTA); + assertEquals(sumPointWeight, calculator.sumWeight(), DELTA); + assertThat(calculator.getDimensionalShapeType(), equalTo(POINT)); + + } + + public void testRoundingErrorAndNormalization() { + double lonA = GeometryTestUtils.randomLon(); + double latA = GeometryTestUtils.randomLat(); + double lonB = randomValueOtherThanMany((l) -> Math.abs(l - lonA) <= GeoUtils.TOLERANCE, GeometryTestUtils::randomLon); + double latB = randomValueOtherThanMany((l) -> Math.abs(l - latA) <= GeoUtils.TOLERANCE, GeometryTestUtils::randomLat); + { + Line line = new Line(new double[]{180.0, 180.0}, new double[]{latA, latB}); + assertThat(new CentroidCalculator(line).getX(), anyOf(equalTo(179.99999999999997), + equalTo(180.0), equalTo(-179.99999999999997))); + } + + { + Line line = new Line(new double[]{-180.0, -180.0}, new double[]{latA, latB}); + assertThat(new CentroidCalculator(line).getX(), anyOf(equalTo(179.99999999999997), + equalTo(180.0), equalTo(-179.99999999999997))); + } + + { + Line line = new Line(new double[]{lonA, lonB}, new double[] { 90.0, 90.0 }); + assertThat(new CentroidCalculator(line).getY(), anyOf(equalTo(90.0), equalTo(89.99999999999999))); + } + + { + Line line = new Line(new double[]{lonA, lonB}, new double[] { -90.0, -90.0 }); + assertThat(new CentroidCalculator(line).getY(), anyOf(equalTo(-90.0), equalTo(-89.99999999999999))); + } + } + + // test that the centroid calculation is agnostic to orientation + public void testPolyonWithHole() { + for (boolean ccwOuter : List.of(true, false)) { + for (boolean ccwInner : List.of(true, false)) { + final LinearRing outer, inner; + if (ccwOuter) { + outer = new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}); + } else { + outer = new LinearRing(new double[]{-50, -50, 50, 50, -50}, new double[]{-50, 50, 50, -50, -50}); + } + if (ccwInner) { + inner = new LinearRing(new double[]{-40, 30, 30, -40, -40}, new double[]{-40, -40, 30, 30, -40}); + } else { + inner = new LinearRing(new double[]{-40, -40, 30, 30, -40}, new double[]{-40, 30, 30, -40, -40}); + } + final double POLY_CENTROID = 4.803921568627451; + CentroidCalculator calculator = new CentroidCalculator(new Polygon(outer, Collections.singletonList(inner))); + assertEquals(POLY_CENTROID, calculator.getX(), DELTA); + assertEquals(POLY_CENTROID, calculator.getY(), DELTA); + assertThat(calculator.sumWeight(), equalTo(5100.0)); + } + } + } + + public void testLineAsClosedPoint() { + double lon = GeometryTestUtils.randomLon(); + double lat = GeometryTestUtils.randomLat(); + CentroidCalculator calculator = new CentroidCalculator(new Line(new double[] {lon, lon}, new double[] { lat, lat})); + assertThat(calculator.getX(), equalTo(GeoUtils.normalizeLon(lon))); + assertThat(calculator.getY(), equalTo(GeoUtils.normalizeLat(lat))); + assertThat(calculator.sumWeight(), equalTo(1.0)); + } + + public void testPolygonAsLine() { + // create a line that traces itself as a polygon + Line sourceLine = GeometryTestUtils.randomLine(false); + double[] x = new double[2 * sourceLine.length() - 1]; + double[] y = new double[2 * sourceLine.length() - 1]; + int idx = 0; + for (int i = 0; i < sourceLine.length(); i++) { + x[idx] = sourceLine.getX(i); + y[idx] = sourceLine.getY(i); + idx += 1; + } + for (int i = sourceLine.length() - 2; i >= 0; i--) { + x[idx] = sourceLine.getX(i); + y[idx] = sourceLine.getY(i); + idx += 1; + } + + Line line = new Line(x, y); + CentroidCalculator lineCalculator = new CentroidCalculator(line); + + Polygon polygon = new Polygon(new LinearRing(x, y)); + CentroidCalculator calculator = new CentroidCalculator(polygon); + + // sometimes precision issues yield non-zero areas. must verify that area is close to zero + if (calculator.getDimensionalShapeType() == POLYGON) { + assertEquals(0.0, calculator.sumWeight(), 1e-10); + } else { + assertThat(calculator.getDimensionalShapeType(), equalTo(LINE)); + assertThat(calculator.getX(), equalTo(lineCalculator.getX())); + assertThat(calculator.getY(), equalTo(lineCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(lineCalculator.compSumWeight.value())); + } + } + + public void testPolygonWithEqualSizedHole() { + Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), + Collections.singletonList(new LinearRing(new double[]{-50, -50, 50, 50, -50}, new double[]{-50, 50, 50, -50, -50}))); + CentroidCalculator calculator = new CentroidCalculator(polyWithHole); + assertThat(calculator.getX(), equalTo(0.0)); + assertThat(calculator.getY(), equalTo(0.0)); + assertThat(calculator.sumWeight(), equalTo(400.0)); + assertThat(calculator.getDimensionalShapeType(), equalTo(LINE)); + } + + public void testPolygonAsPoint() { + Point point = GeometryTestUtils.randomPoint(false); + Polygon polygon = new Polygon(new LinearRing(new double[] { point.getX(), point.getX(), point.getX(), point.getX() }, + new double[] { point.getY(), point.getY(), point.getY(), point.getY() })); + CentroidCalculator calculator = new CentroidCalculator(polygon); + assertThat(calculator.getX(), equalTo(GeoUtils.normalizeLon(point.getX()))); + assertThat(calculator.getY(), equalTo(GeoUtils.normalizeLat(point.getY()))); + assertThat(calculator.sumWeight(), equalTo(1.0)); + assertThat(calculator.getDimensionalShapeType(), equalTo(POINT)); + } + + public void testGeometryCollection() { + int numPoints = randomIntBetween(0, 3); + int numLines = randomIntBetween(0, 3); + int numPolygons = randomIntBetween(0, 3); + + if (numPoints == 0 && numLines == 0 && numPolygons == 0) { + numPoints = 1; + numLines = 1; + numPolygons = 1; + } + List shapes = new ArrayList<>(); + for (int i = 0; i < numPoints; i++) { + if (randomBoolean()) { + shapes.add(GeometryTestUtils.randomPoint(false)); + } else { + shapes.add(GeometryTestUtils.randomMultiPoint(false)); + } + } + for (int i = 0; i < numLines; i++) { + if (randomBoolean()) { + shapes.add(GeometryTestUtils.randomLine(false)); + } else { + shapes.add(GeometryTestUtils.randomMultiLine(false)); + } + } + for (int i = 0; i < numPolygons; i++) { + if (randomBoolean()) { + shapes.add(GeometryTestUtils.randomPolygon(false)); + } else { + shapes.add(GeometryTestUtils.randomMultiPolygon(false)); + } + } + + DimensionalShapeType dimensionalShapeType = numPolygons > 0 ? POLYGON : numLines > 0 ? LINE : POINT; + + // addFromCalculator is only adding from shapes with the highest dimensionalShapeType + CentroidCalculator addFromCalculator = null; + for (Geometry shape : shapes) { + if ((shape.type() == ShapeType.MULTIPOLYGON || shape.type() == ShapeType.POLYGON) || + (dimensionalShapeType == LINE && (shape.type() == ShapeType.LINESTRING || shape.type() == ShapeType.MULTILINESTRING)) || + (dimensionalShapeType == POINT && (shape.type() == ShapeType.POINT || shape.type() == ShapeType.MULTIPOINT))) { + if (addFromCalculator == null) { + addFromCalculator = new CentroidCalculator(shape); + } else { + addFromCalculator.addFrom(new CentroidCalculator(shape)); + } + } + } + + // shuffle + if (randomBoolean()) { + Collections.shuffle(shapes, random()); + } else if (randomBoolean()) { + Collections.reverse(shapes); + } + + GeometryCollection collection = new GeometryCollection<>(shapes); + CentroidCalculator calculator = new CentroidCalculator(collection); + + assertThat(addFromCalculator.getDimensionalShapeType(), equalTo(dimensionalShapeType)); + assertThat(calculator.getDimensionalShapeType(), equalTo(dimensionalShapeType)); + assertEquals(calculator.getX(), addFromCalculator.getX(), DELTA); + assertEquals(calculator.getY(), addFromCalculator.getY(), DELTA); + assertEquals(calculator.sumWeight(), addFromCalculator.sumWeight(), DELTA); + } + + public void testAddFrom() { + Point point = GeometryTestUtils.randomPoint(false); + Line line = GeometryTestUtils.randomLine(false); + Polygon polygon = GeometryTestUtils.randomPolygon(false); + + // point add point + { + CentroidCalculator calculator = new CentroidCalculator(point); + calculator.addFrom(new CentroidCalculator(point)); + assertThat(calculator.compSumX.value(), equalTo(2 * point.getX())); + assertThat(calculator.compSumY.value(), equalTo(2 * point.getY())); + assertThat(calculator.sumWeight(), equalTo(2.0)); + } + + // point add line/polygon + { + CentroidCalculator lineCalculator = new CentroidCalculator(line); + CentroidCalculator calculator = new CentroidCalculator(point); + calculator.addFrom(lineCalculator); + assertThat(calculator.getX(), equalTo(lineCalculator.getX())); + assertThat(calculator.getY(), equalTo(lineCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(lineCalculator.sumWeight())); + } + + // line add point + { + CentroidCalculator lineCalculator = new CentroidCalculator(line); + CentroidCalculator calculator = new CentroidCalculator(line); + calculator.addFrom(new CentroidCalculator(point)); + assertThat(calculator.getX(), equalTo(lineCalculator.getX())); + assertThat(calculator.getY(), equalTo(lineCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(lineCalculator.sumWeight())); + } + + // line add line + { + CentroidCalculator lineCalculator = new CentroidCalculator(line); + CentroidCalculator calculator = new CentroidCalculator(line); + calculator.addFrom(lineCalculator); + assertEquals(2 * lineCalculator.compSumX.value(), calculator.compSumX.value(), DELTA); + assertEquals(2 * lineCalculator.compSumY.value(), calculator.compSumY.value(), DELTA); + assertEquals(2 * lineCalculator.sumWeight(), calculator.sumWeight(), DELTA); + } + + // line add polygon + { + CentroidCalculator polygonCalculator = new CentroidCalculator(polygon); + CentroidCalculator calculator = new CentroidCalculator(line); + calculator.addFrom(polygonCalculator); + assertThat(calculator.getX(), equalTo(polygonCalculator.getX())); + assertThat(calculator.getY(), equalTo(polygonCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(calculator.sumWeight())); + } + + // polygon add point/line + { + CentroidCalculator polygonCalculator = new CentroidCalculator(polygon); + CentroidCalculator calculator = new CentroidCalculator(polygon); + calculator.addFrom(new CentroidCalculator(randomBoolean() ? point : line)); + assertThat(calculator.getX(), equalTo(polygonCalculator.getX())); + assertThat(calculator.getY(), equalTo(polygonCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(calculator.sumWeight())); + } + + // polygon add polygon + { + CentroidCalculator polygonCalculator = new CentroidCalculator(polygon); + CentroidCalculator calculator = new CentroidCalculator(polygon); + calculator.addFrom(polygonCalculator); + assertThat(calculator.compSumX.value(), equalTo(2 * polygonCalculator.compSumX.value())); + assertThat(calculator.compSumY.value(), equalTo(2 * polygonCalculator.compSumY.value())); + assertThat(calculator.sumWeight(), equalTo(2 * polygonCalculator.sumWeight())); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java b/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java new file mode 100644 index 0000000000000..9966f87dc97e2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class DimensionalShapeTypeTests extends ESTestCase { + + public void testValidOrdinals() { + assertThat(DimensionalShapeType.values().length, equalTo(3)); + assertThat(DimensionalShapeType.POINT.ordinal(), equalTo(0)); + assertThat(DimensionalShapeType.LINE.ordinal(), equalTo(1)); + assertThat(DimensionalShapeType.POLYGON.ordinal(), equalTo(2)); + } + + public void testSerialization() { + for (DimensionalShapeType type : DimensionalShapeType.values()) { + ByteBuffersDataOutput out = new ByteBuffersDataOutput(); + type.writeTo(out); + ByteArrayDataInput input = new ByteArrayDataInput(out.toArrayCopy()); + assertThat(DimensionalShapeType.readFrom(input), equalTo(type)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java new file mode 100644 index 0000000000000..979f445ee8c5f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class ExtentTests extends ESTestCase { + + public void testFromPoint() { + int x = randomFrom(-1, 0, 1); + int y = randomFrom(-1, 0, 1); + Extent extent = Extent.fromPoint(x, y); + assertThat(extent.minX(), equalTo(x)); + assertThat(extent.maxX(), equalTo(x)); + assertThat(extent.minY(), equalTo(y)); + assertThat(extent.maxY(), equalTo(y)); + } + + public void testAddRectangle() { + Extent extent = new Extent(); + int bottomLeftX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(-175); + int bottomLeftY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(-10); + int topRightX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(-170); + int topRightY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(10); + extent.addRectangle(bottomLeftX, bottomLeftY, topRightX, topRightY); + assertThat(extent.minX(), equalTo(bottomLeftX)); + assertThat(extent.maxX(), equalTo(topRightX)); + assertThat(extent.minY(), equalTo(bottomLeftY)); + assertThat(extent.maxY(), equalTo(topRightY)); + int bottomLeftX2 = GeoShapeCoordinateEncoder.INSTANCE.encodeX(170); + int bottomLeftY2 = GeoShapeCoordinateEncoder.INSTANCE.encodeY(-20); + int topRightX2 = GeoShapeCoordinateEncoder.INSTANCE.encodeX(175); + int topRightY2 = GeoShapeCoordinateEncoder.INSTANCE.encodeY(20); + extent.addRectangle(bottomLeftX2, bottomLeftY2, topRightX2, topRightY2); + assertThat(extent.minX(), equalTo(bottomLeftX)); + assertThat(extent.maxX(), equalTo(topRightX2)); + assertThat(extent.minY(), equalTo(bottomLeftY2)); + assertThat(extent.maxY(), equalTo(topRightY2)); + } + + public void testSerialize() throws IOException { + for (int i =0; i < 100; i++) { + Extent extent = randomExtent(); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); + extent.writeCompressed(output); + BytesRef bytesRef = new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size())); + ByteArrayDataInput input = new ByteArrayDataInput(); + input.reset(bytesRef.bytes, bytesRef.offset, bytesRef.length); + Extent copyExtent = new Extent(); + Extent.readFromCompressed(input, copyExtent); + assertEquals(extent, copyExtent); + } + } + + private Extent randomExtent() { + Extent extent = new Extent(); + int numberPoints = random().nextBoolean() ? 1 : randomIntBetween(2, 10); + for (int i =0; i < numberPoints; i++) { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + while (rectangle.getMinX() > rectangle.getMaxX()) { + rectangle = GeometryTestUtils.randomRectangle(); + } + int bottomLeftX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMinX()); + int bottomLeftY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMinY()); + int topRightX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMaxX()); + int topRightY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMaxY()); + extent.addRectangle(bottomLeftX, bottomLeftY, topRightX, topRightY); + } + return extent; + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoderTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoderTests.java new file mode 100644 index 0000000000000..9a6e822a198da --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoderTests.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.geo.GeoEncodingUtils; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.elasticsearch.common.geo.GeoShapeCoordinateEncoder.INSTANCE; + + +public class GeoShapeCoordinateEncoderTests extends ESTestCase { + + public void testLongitude() { + double randomLon = randomDoubleBetween(-180, 180, true); + double randomInvalidLon = randomFrom(randomDoubleBetween(-1000, -180.01, true), + randomDoubleBetween(180.01, 1000, true)); + + assertThat(INSTANCE.encodeX(Double.POSITIVE_INFINITY), equalTo(Integer.MAX_VALUE)); + assertThat(INSTANCE.encodeX(Double.NEGATIVE_INFINITY), equalTo(Integer.MIN_VALUE)); + int encodedLon = INSTANCE.encodeX(randomLon); + assertThat(encodedLon, equalTo(GeoEncodingUtils.encodeLongitude(randomLon))); + Exception e = expectThrows(IllegalArgumentException.class, () -> GeoShapeCoordinateEncoder.INSTANCE.encodeX(randomInvalidLon)); + assertThat(e.getMessage(), endsWith("must be between -180.0 and 180.0")); + + assertThat(INSTANCE.decodeX(encodedLon), closeTo(randomLon, 0.0001)); + assertThat(INSTANCE.decodeX(Integer.MAX_VALUE), closeTo(180, 0.00001)); + assertThat(INSTANCE.decodeX(Integer.MIN_VALUE), closeTo(-180, 0.00001)); + } + + public void testLatitude() { + double randomLat = randomDoubleBetween(-90, 90, true); + double randomInvalidLat = randomFrom(randomDoubleBetween(-1000, -90.01, true), + randomDoubleBetween(90.01, 1000, true)); + + assertThat(INSTANCE.encodeY(Double.POSITIVE_INFINITY), equalTo(Integer.MAX_VALUE)); + assertThat(INSTANCE.encodeY(Double.NEGATIVE_INFINITY), equalTo(Integer.MIN_VALUE)); + int encodedLat = INSTANCE.encodeY(randomLat); + assertThat(encodedLat, equalTo(GeoEncodingUtils.encodeLatitude(randomLat))); + Exception e = expectThrows(IllegalArgumentException.class, () -> GeoShapeCoordinateEncoder.INSTANCE.encodeY(randomInvalidLat)); + assertThat(e.getMessage(), endsWith("must be between -90.0 and 90.0")); + + assertThat(INSTANCE.decodeY(encodedLat), closeTo(randomLat, 0.0001)); + assertThat(INSTANCE.decodeY(Integer.MAX_VALUE), closeTo(90, 0.00001)); + assertThat(INSTANCE.decodeY(Integer.MIN_VALUE), closeTo(-90, 0.00001)); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java new file mode 100644 index 0000000000000..2160340dc0a64 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -0,0 +1,100 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.index.mapper.GeoShapeIndexer; + + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +public class GeoTestUtils { + + public static void assertRelation(GeoRelation expectedRelation, TriangleTreeReader reader, Extent extent) throws IOException { + GeoRelation actualRelation = reader.relateTile(extent.minX(), extent.minY(), extent.maxX(), extent.maxY()); + assertThat(actualRelation, equalTo(expectedRelation)); + } + + public static ShapeField.DecodedTriangle[] toDecodedTriangles(Geometry geometry) throws IOException { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + List fields = indexer.indexShape(null, geometry); + ShapeField.DecodedTriangle[] triangles = new ShapeField.DecodedTriangle[fields.size()]; + final byte[] scratch = new byte[7 * Integer.BYTES]; + for (int i = 0; i < fields.size(); i++) { + BytesRef bytesRef = fields.get(i).binaryValue(); + assert bytesRef.length == 7 * Integer.BYTES; + System.arraycopy(bytesRef.bytes, bytesRef.offset, scratch, 0, 7 * Integer.BYTES); + ShapeField.decodeTriangle(scratch, triangles[i] = new ShapeField.DecodedTriangle()); + } + return triangles; + } + + public static TriangleTreeReader triangleTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { + ShapeField.DecodedTriangle[] triangles = toDecodedTriangles(geometry); + TriangleTreeWriter writer = new TriangleTreeWriter(Arrays.asList(triangles), encoder, new CentroidCalculator(geometry)); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); + writer.writeTo(output); + TriangleTreeReader reader = new TriangleTreeReader(encoder); + reader.reset(new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size()))); + return reader; + } + + public static double encodeDecodeLat(double lat) { + return GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); + } + + public static double encodeDecodeLon(double lon) { + return GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); + } + + public static String toGeoJsonString(Geometry geometry) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + GeoJson.toXContent(geometry, builder, ToXContent.EMPTY_PARAMS); + return XContentHelper.convertToJson(BytesReference.bytes(builder), true, false, XContentType.JSON); + } + + public static Geometry fromGeoJsonString(String geoJson) throws Exception { + XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + new BytesArray(geoJson), XContentType.JSON); + parser.nextToken(); + Geometry geometry = new GeometryParser(true, true, true).parse(parser); + return new GeoShapeIndexer(true, "indexer").prepareForIndexing(geometry); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/TestCoordinateEncoder.java b/server/src/test/java/org/elasticsearch/common/geo/TestCoordinateEncoder.java new file mode 100644 index 0000000000000..222bfeff4e526 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/TestCoordinateEncoder.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + + +/** + * {@link CoordinateEncoder} used for tests that is an identity-encoder-decoder + */ +public class TestCoordinateEncoder implements CoordinateEncoder { + + public static final TestCoordinateEncoder INSTANCE = new TestCoordinateEncoder(); + + @Override + public int encodeX(double x) { + return (int) x; + } + + @Override + public int encodeY(double y) { + return (int) y; + } + + @Override + public double decodeX(int x) { + return x; + } + + @Override + public double decodeY(int y) { + return y; + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java new file mode 100644 index 0000000000000..f7561508f831d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -0,0 +1,362 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; +import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; +import static org.elasticsearch.geo.GeometryTestUtils.fold; +import static org.elasticsearch.geo.GeometryTestUtils.randomLine; +import static org.elasticsearch.geo.GeometryTestUtils.randomMultiLine; +import static org.elasticsearch.geo.GeometryTestUtils.randomMultiPoint; +import static org.elasticsearch.geo.GeometryTestUtils.randomMultiPolygon; +import static org.elasticsearch.geo.GeometryTestUtils.randomPoint; +import static org.elasticsearch.geo.GeometryTestUtils.randomPolygon; +import static org.elasticsearch.geo.GeometryTestUtils.randomRectangle; +import static org.hamcrest.Matchers.equalTo; + +public class TriangleTreeTests extends ESTestCase { + + @SuppressWarnings("unchecked") + public void testDimensionalShapeType() throws IOException { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + assertDimensionalShapeType(randomPoint(false), DimensionalShapeType.POINT); + assertDimensionalShapeType(randomMultiPoint(false), DimensionalShapeType.POINT); + assertDimensionalShapeType(randomLine(false), DimensionalShapeType.LINE); + assertDimensionalShapeType(randomMultiLine(false), DimensionalShapeType.LINE); + Geometry randoPoly = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { + try { + Geometry newGeo = indexer.prepareForIndexing(g); + return newGeo.type() != ShapeType.POLYGON; + } catch (Exception e) { + return true; + } + }, () -> randomPolygon(false))); + assertDimensionalShapeType(randoPoly, DimensionalShapeType.POLYGON); + assertDimensionalShapeType(indexer.prepareForIndexing(randomMultiPolygon(false)), DimensionalShapeType.POLYGON); + assertDimensionalShapeType(randomRectangle(), DimensionalShapeType.POLYGON); + assertDimensionalShapeType(randomFrom( + new GeometryCollection<>(List.of(randomPoint(false))), + new GeometryCollection<>(List.of(randomMultiPoint(false))), + new GeometryCollection<>(Collections.singletonList( + new GeometryCollection<>(List.of(randomPoint(false), randomMultiPoint(false)))))) + , DimensionalShapeType.POINT); + assertDimensionalShapeType(randomFrom( + new GeometryCollection<>(List.of(randomPoint(false), randomLine(false))), + new GeometryCollection<>(List.of(randomMultiPoint(false), randomMultiLine(false))), + new GeometryCollection<>(Collections.singletonList( + new GeometryCollection<>(List.of(randomPoint(false), randomLine(false)))))) + , DimensionalShapeType.LINE); + assertDimensionalShapeType(randomFrom( + new GeometryCollection<>(List.of(randomPoint(false), indexer.prepareForIndexing(randomLine(false)), + indexer.prepareForIndexing(randomPolygon(false)))), + new GeometryCollection<>(List.of(randomMultiPoint(false), indexer.prepareForIndexing(randomMultiPolygon(false)))), + new GeometryCollection<>(Collections.singletonList( + new GeometryCollection<>(List.of(indexer.prepareForIndexing(randomLine(false)), + indexer.prepareForIndexing(randomPolygon(false))))))) + , DimensionalShapeType.POLYGON); + } + + + public void testRectangleShape() throws IOException { + for (int i = 0; i < 1000; i++) { + int minX = randomIntBetween(-40, -1); + int maxX = randomIntBetween(1, 40); + int minY = randomIntBetween(-40, -1); + int maxY = randomIntBetween(1, 40); + Geometry rectangle = new Rectangle(minX, maxX, maxY, minY); + TriangleTreeReader reader = triangleTreeReader(rectangle, GeoShapeCoordinateEncoder.INSTANCE); + + Extent expectedExtent = getExtentFromBox(minX, minY, maxX, maxY); + assertThat(expectedExtent, equalTo(reader.getExtent())); + // centroid is calculated using original double values but then loses precision as it is serialized as an integer + int encodedCentroidX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(((double) minX + maxX) / 2); + int encodedCentroidY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(((double) minY + maxY) / 2); + assertEquals(GeoShapeCoordinateEncoder.INSTANCE.decodeX(encodedCentroidX), reader.getCentroidX(), 0.0000001); + assertEquals(GeoShapeCoordinateEncoder.INSTANCE.decodeY(encodedCentroidY), reader.getCentroidY(), 0.0000001); + + // box-query touches bottom-left corner + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), + minY - randomIntBetween(1, 90 + minY), minX, minY)); + // box-query touches bottom-right corner + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(maxX, minY - randomIntBetween(1, 90 + minY), + maxX + randomIntBetween(1, 180 - maxX), minY)); + // box-query touches top-right corner + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(maxX, maxY, maxX + randomIntBetween(1, 180 - maxX), + maxY + randomIntBetween(1, 90 - maxY))); + // box-query touches top-left corner + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), maxY, minX, + maxY + randomIntBetween(1, 90 - maxY))); + + // box-query fully-enclosed inside rectangle + assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(3 * (minX + maxX) / 4, 3 * (minY + maxY) / 4, + 3 * (maxX + minX) / 4, 3 * (maxY + minY) / 4)); + // box-query fully-contains poly + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), + minY - randomIntBetween(1, 90 + minY), maxX + randomIntBetween(1, 180 - maxX), + maxY + randomIntBetween(1, 90 - maxY))); + // box-query half-in-half-out-right + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(3 * (minX + maxX) / 4, 3 * (minY + maxY) / 4, + maxX + randomIntBetween(1, 90 - maxY), 3 * (maxY + minY) / 4)); + // box-query half-in-half-out-left + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), + 3 * (minY + maxY) / 4, 3 * (maxX + minX) / 4, 3 * (maxY + minY) / 4)); + // box-query half-in-half-out-top + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(3 * (minX + maxX) / 4, 3 * (minY + maxY) / 4, + maxX + randomIntBetween(1, 180 - maxX), maxY + randomIntBetween(1, 90 - maxY))); + // box-query half-in-half-out-bottom + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(3 * (minX + maxX) / 4, + minY - randomIntBetween(1, 90 + minY), maxX + randomIntBetween(1, 180 - maxX), + 3 * (maxY + minY) / 4)); + + // box-query outside to the right + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(maxX + randomIntBetween(1, 180 - maxX), minY, + maxX + randomIntBetween(1, 180 - maxX), maxY)); + // box-query outside to the left + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(maxX - randomIntBetween(1, 180 - maxX), minY, + minX - randomIntBetween(1, 180 + minX), maxY)); + // box-query outside to the top + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(minX, maxY + randomIntBetween(1, 90 - maxY), maxX, + maxY + randomIntBetween(1, 90 - maxY))); + // box-query outside to the bottom + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(minX, minY - randomIntBetween(1, 90 + minY), maxX, + minY - randomIntBetween(1, 90 + minY))); + } + } + + public void testPacManPolygon() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, -5, -9, -10, -9, 0, 9, 10, 9, 5, 0}; + + // test cell crossing poly + Polygon pacMan = new Polygon(new LinearRing(py, px), Collections.emptyList()); + TriangleTreeReader reader = triangleTreeReader(pacMan, TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(-5, -6, 2, -2)); + } + + // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon + public void testPolygonWithHole() throws Exception { + Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), + Collections.singletonList(new LinearRing(new double[]{-10, 10, 10, -10, -10}, new double[]{-10, -10, 10, 10, -10}))); + + TriangleTreeReader reader = triangleTreeReader(polyWithHole, GeoShapeCoordinateEncoder.INSTANCE); + + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(6, -6, 6, -6)); // in the hole + assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(25, -25, 25, -25)); // on the mainland + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(51, 51, 52, 52)); // outside of mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-60, -60, 60, 60)); // enclosing us completely + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(49, 49, 51, 51)); // overlapping the mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(9, 9, 11, 11)); // overlapping the hole + } + + public void testCombPolygon() throws Exception { + double[] px = {0, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 0, 0}; + double[] py = {0, 0, 20, 20, 0, 0, 20, 20, 0, 0, 30, 30, 0}; + + double[] hx = {21, 21, 29, 29, 21}; + double[] hy = {1, 20, 20, 1, 1}; + + Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); + TriangleTreeReader reader = triangleTreeReader(polyWithHole, GeoShapeCoordinateEncoder.INSTANCE); + // test cell crossing poly + assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(5, 10, 5, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(15, 10, 15, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(25, 10, 25, 10)); + } + + public void testPacManClosedLineString() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // test cell crossing poly + TriangleTreeReader reader = triangleTreeReader(new Line(px, py), GeoShapeCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(-5, -6, 2, -2)); + } + + public void testPacManLineString() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; + + // test cell crossing poly + TriangleTreeReader reader = triangleTreeReader(new Line(px, py), GeoShapeCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(-5, -6, 2, -2)); + } + + public void testPacManPoints() throws Exception { + // pacman + List points = Arrays.asList( + new Point(0, 0), + new Point(5, 10), + new Point(9, 10), + new Point(10, 0), + new Point(9, -8), + new Point(0, -10), + new Point(-9, -8), + new Point(-10, 0), + new Point(-9, 10), + new Point(-5, 10) + ); + + + // candidate intersects cell + int xMin = 0; + int xMax = 11; + int yMin = -10; + int yMax = 9; + + // test cell crossing poly + TriangleTreeReader reader = triangleTreeReader(new MultiPoint(points), GeoShapeCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(xMin, yMin, xMax, yMax)); + } + + public void testRandomMultiLineIntersections() throws IOException { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + MultiLine geometry = randomMultiLine(false); + geometry = (MultiLine) indexer.prepareForIndexing(geometry); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + Extent readerExtent = reader.getExtent(); + + for (Line line : geometry) { + Extent lineExtent = triangleTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); + if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE + && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, + lineExtent.maxX() + 1, lineExtent.maxY() + 1)); + } + } + + // extent that fully encloses the MultiLine + assertRelation(GeoRelation.QUERY_CROSSES, reader, reader.getExtent()); + if (readerExtent.minX() != Integer.MIN_VALUE && readerExtent.maxX() != Integer.MAX_VALUE + && readerExtent.minY() != Integer.MIN_VALUE && readerExtent.maxY() != Integer.MAX_VALUE) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, + readerExtent.maxX() + 1, readerExtent.maxY() + 1)); + } + + } + + public void testRandomPolygonIntersection() throws IOException { + int testPointCount = randomIntBetween(50, 100); + Point[] testPoints = new Point[testPointCount]; + double extentSize = randomDoubleBetween(1, 10, true); + boolean[] intersects = new boolean[testPointCount]; + for (int i = 0; i < testPoints.length; i++) { + testPoints[i] = randomPoint(false); + } + + Geometry geometry = randomMultiPolygon(false); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry preparedGeometry = indexer.prepareForIndexing(geometry); + + for (int i = 0; i < testPointCount; i++) { + int cur = i; + intersects[cur] = fold(preparedGeometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); + } + + for (int i = 0; i < testPointCount; i++) { + assertEquals(intersects[i], intersects(preparedGeometry, testPoints[i], extentSize)); + } + } + + private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) { + int xMin = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.max(x - extentSize, -180.0)); + int xMax = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.min(x + extentSize, 180.0)); + int yMin = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.max(y - extentSize, -90)); + int yMax = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.min(y + extentSize, 90)); + return Extent.fromPoints(xMin, yMin, xMax, yMax); + } + + private static Extent getExtentFromBox(double bottomLeftX, double bottomLeftY, double topRightX, double topRightY) { + return Extent.fromPoints(GeoShapeCoordinateEncoder.INSTANCE.encodeX(bottomLeftX), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(bottomLeftY), + GeoShapeCoordinateEncoder.INSTANCE.encodeX(topRightX), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(topRightY)); + + } + + private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { + + Extent bufferBounds = bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize); + GeoRelation relation = triangleTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) + .relateTile(bufferBounds.minX(), bufferBounds.minY(), bufferBounds.maxX(), bufferBounds.maxY()); + return relation == GeoRelation.QUERY_CROSSES || relation == GeoRelation.QUERY_INSIDE; + } + + private static Geometry randomGeometryTreeGeometry() { + return randomGeometryTreeGeometry(0); + } + + private static Geometry randomGeometryTreeGeometry(int level) { + @SuppressWarnings("unchecked") Function geometry = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPoint, + level < 3 ? (b) -> randomGeometryTreeCollection(level + 1) : GeometryTestUtils::randomPoint // don't build too deep + ); + return geometry.apply(false); + } + + private static Geometry randomGeometryTreeCollection(int level) { + int size = ESTestCase.randomIntBetween(1, 10); + List shapes = new ArrayList<>(); + for (int i = 0; i < size; i++) { + shapes.add(randomGeometryTreeGeometry(level)); + } + return new GeometryCollection<>(shapes); + } + + private static void assertDimensionalShapeType(Geometry geometry, DimensionalShapeType expected) throws IOException { + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + assertThat(reader.getDimensionalShapeType(), equalTo(expected)); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java index 0d4f142785484..6095b0f772834 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java @@ -33,6 +33,7 @@ public class PolygonBuilderTests extends AbstractShapeBuilderTestCase> getPlugins() { return pluginList(InternalSettingsPlugin.class); @@ -60,7 +67,28 @@ public void testDefaultConfiguration() throws IOException { GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value())); - assertThat(geoShapeFieldMapper.fieldType.hasDocValues(), equalTo(false)); + assertFalse(geoShapeFieldMapper.docValues().explicit()); + assertTrue(geoShapeFieldMapper.docValues().value()); + assertTrue(geoShapeFieldMapper.fieldType().hasDocValues()); + } + + public void testDefaultDocValueConfigurationOnPre8() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject() + .endObject().endObject()); + + Version oldVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0); + DocumentMapper defaultMapper = createIndex("test", settings(oldVersion).build()).mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + assertFalse(geoShapeFieldMapper.docValues().explicit()); + assertFalse(geoShapeFieldMapper.docValues().value()); + assertFalse(geoShapeFieldMapper.fieldType().hasDocValues()); } /** @@ -214,6 +242,45 @@ public void testIgnoreMalformedParsing() throws IOException { assertThat(ignoreMalformed.value(), equalTo(false)); } + /** + * Test that doc_values parameter correctly parses + */ + public void testDocValues() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("doc_values", true) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().explicit()); + assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().value()); + boolean hasDocValues = ((GeoShapeFieldMapper)fieldMapper).fieldType().hasDocValues(); + assertTrue(hasDocValues); + + // explicit false doc_values + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("doc_values", "false") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().explicit()); + assertFalse(((GeoShapeFieldMapper)fieldMapper).docValues().value()); + hasDocValues = ((GeoShapeFieldMapper)fieldMapper).fieldType().hasDocValues(); + assertFalse(hasDocValues); + } private void assertFieldWarnings(String... fieldNames) { String[] warnings = new String[fieldNames.length]; @@ -283,9 +350,26 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"orientation\":\"" + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":true")); } } + public void testSerializeDocValues() throws IOException { + boolean docValues = randomBoolean(); + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("doc_values", docValues) + .endObject().endObject() + .endObject().endObject()); + DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((GeoShapeFieldMapper) mapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":" + docValues)); + } + public String toXContentString(GeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); ToXContent.Params params; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java index aaabf3f9edbf0..2b17ba8a55fa8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -83,6 +83,10 @@ public void testDefaultConfiguration() throws IOException { equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.DISTANCE_ERROR_PCT)); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(LegacyGeoShapeFieldMapper.Defaults.ORIENTATION.value())); + assertThat(geoShapeFieldMapper.docValues(), + equalTo(LegacyGeoShapeFieldMapper.Defaults.DOC_VALUES)); + assertThat(geoShapeFieldMapper.fieldType().hasDocValues(), + equalTo(LegacyGeoShapeFieldMapper.Defaults.DOC_VALUES.value())); assertFieldWarnings("strategy"); } @@ -598,6 +602,7 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); assertTrue(serialized, serialized.contains("\"tree_levels\":21")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -610,6 +615,7 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); assertTrue(serialized, serialized.contains("\"tree_levels\":9")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -623,6 +629,7 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertFalse(serialized, serialized.contains("\"precision\":")); assertTrue(serialized, serialized.contains("\"tree_levels\":6")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -636,6 +643,7 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); assertFalse(serialized, serialized.contains("\"tree_levels\":")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -650,10 +658,29 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); assertTrue(serialized, serialized.contains("\"tree_levels\":5")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } assertFieldWarnings("tree", "tree_levels", "precision"); } + public void testSerializeDocValues() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("doc_values", false) + .endObject().endObject() + .endObject().endObject()); + DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) mapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); + + assertFieldWarnings("tree"); + } + public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -702,6 +729,43 @@ public void testPointsOnlyFalseWithTermStrategy() throws Exception { assertFieldWarnings("tree", "precision", "strategy", "points_only"); } + /** + * Test that doc_values parameter correctly parses + */ + public void testDocValues() throws IOException { + String trueMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("doc_values", true) + .endObject().endObject() + .endObject().endObject()); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(trueMapping))); + assertThat(e.getMessage(), equalTo("geo_shape field [location] indexed using prefix-trees do not support doc_values")); + + // explicit false doc_values + String falseMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("doc_values", "false") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(falseMapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + assertTrue(((LegacyGeoShapeFieldMapper) fieldMapper).docValues().explicit()); + assertFalse(((LegacyGeoShapeFieldMapper) fieldMapper).docValues().value()); + assertFalse(((LegacyGeoShapeFieldMapper) fieldMapper).fieldType().hasDocValues()); + + assertFieldWarnings("tree"); + } + public void testDisallowExpensiveQueries() throws IOException { QueryShardContext queryShardContext = mock(QueryShardContext.class); when(queryShardContext.allowExpensiveQueries()).thenReturn(false); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java index 61f6c60424671..6d11f93616f41 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java @@ -23,12 +23,16 @@ import com.carrotsearch.hppc.cursors.ObjectIntCursor; import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.filter.Filter; @@ -43,6 +47,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.geometry.utils.Geohash.PRECISION; import static org.elasticsearch.geometry.utils.Geohash.stringEncode; @@ -52,44 +57,46 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; @ESIntegTestCase.SuiteScopeTestCase public class GeoHashGridIT extends ESIntegTestCase { + private static WellKnownText WKT = new WellKnownText(false, new GeographyValidator(true)); @Override protected boolean forbidPrivateIndexSettings() { return false; } - private Version version = VersionUtils.randomIndexCompatibleVersion(random()); - static ObjectIntMap expectedDocCountsForGeoHash = null; static ObjectIntMap multiValuedExpectedDocCountsForGeoHash = null; static int numDocs = 100; static String smallestGeoHash = null; - private static IndexRequestBuilder indexCity(String index, String name, List latLon) throws Exception { + private static IndexRequestBuilder indexCity(String index, String name, List points) throws Exception { XContentBuilder source = jsonBuilder().startObject().field("city", name); - if (latLon != null) { - source = source.field("location", latLon); + if (points != null) { + List latLonAsStrings = points.stream().map(ll -> ll.getLat() + ", " + ll.getLon()).collect(Collectors.toList()); + source = source.field("location", latLonAsStrings); + source = source.field("location_as_shape", points.isEmpty() ? null : WKT.toWKT(new MultiPoint(points))); } source = source.endObject(); return client().prepareIndex(index).setSource(source); } - private static IndexRequestBuilder indexCity(String index, String name, String latLon) throws Exception { - return indexCity(index, name, Arrays.asList(latLon)); + private static IndexRequestBuilder indexCity(String index, String name, double lng, double lat) throws Exception { + return indexCity(index, name, Arrays.asList(new Point(lng, lat))); } @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx_unmapped"); - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - - assertAcked(prepareCreate("idx").setSettings(settings) - .setMapping("location", "type=geo_point", "city", "type=keyword")); + assertAcked(prepareCreate("idx") + .setMapping("location", "type=geo_point", "location_as_shape", "type=geo_shape", "city", "type=keyword")); + assertAcked(prepareCreate("idx7x", settings(VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0))) + .setMapping("location", "type=geo_point", "location_as_shape", "type=geo_shape", "city", "type=keyword")); List cities = new ArrayList<>(); Random random = random(); @@ -100,7 +107,8 @@ public void setupSuiteScopeCluster() throws Exception { double lng = (360d * random.nextDouble()) - 180d; String randomGeoHash = stringEncode(lng, lat, PRECISION); //Index at the highest resolution - cities.add(indexCity("idx", randomGeoHash, lat + ", " + lng)); + cities.add(indexCity("idx", randomGeoHash, lng, lat)); + cities.add(indexCity("idx7x", randomGeoHash, lng, lat)); expectedDocCountsForGeoHash.put(randomGeoHash, expectedDocCountsForGeoHash.getOrDefault(randomGeoHash, 0) + 1); //Update expected doc counts for all resolutions.. for (int precision = PRECISION - 1; precision > 0; precision--) { @@ -113,19 +121,19 @@ public void setupSuiteScopeCluster() throws Exception { } indexRandom(true, cities); - assertAcked(prepareCreate("multi_valued_idx").setSettings(settings) + assertAcked(prepareCreate("multi_valued_idx").setSettings(settings(VersionUtils.randomIndexCompatibleVersion(random()))) .setMapping("location", "type=geo_point", "city", "type=keyword")); cities = new ArrayList<>(); multiValuedExpectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); for (int i = 0; i < numDocs; i++) { final int numPoints = random.nextInt(4); - List points = new ArrayList<>(); + List points = new ArrayList<>(); Set geoHashes = new HashSet<>(); for (int j = 0; j < numPoints; ++j) { double lat = (180d * random.nextDouble()) - 90d; double lng = (360d * random.nextDouble()) - 180d; - points.add(lat + "," + lng); + points.add(new Point(lng, lat)); // Update expected doc counts for all resolutions.. for (int precision = PRECISION; precision > 0; precision--) { final String geoHash = stringEncode(lng, lat, precision); @@ -142,6 +150,42 @@ public void setupSuiteScopeCluster() throws Exception { ensureSearchable(); } + public void test7xIndexOnly() { + SearchPhaseExecutionException exception = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("idx7x") + .addAggregation(geohashGrid("aggName").field("location_as_shape")) + .get()); + assertNotNull(exception.getRootCause()); + assertThat(exception.getRootCause().getMessage(), + equalTo("Can't load fielddata on [location_as_shape] because fielddata is unsupported on fields of type [geo_shape]." + + " Use doc values instead.")); + } + + public void test7xIndexWith8Index() { + int precision = randomIntBetween(1, PRECISION); + SearchResponse response = client().prepareSearch("idx", "idx7x") + .addAggregation(geohashGrid("aggName").field("location_as_shape").precision(precision)) + .get(); + assertThat(response.status(), equalTo(RestStatus.OK)); + assertThat(response.getSuccessfulShards(), lessThan(response.getTotalShards())); + GeoGrid geoGrid = response.getAggregations().get("aggName"); + List buckets = geoGrid.getBuckets(); + Object[] propertiesKeys = (Object[]) ((InternalAggregation)geoGrid).getProperty("_key"); + Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)geoGrid).getProperty("_count"); + for (int i = 0; i < buckets.size(); i++) { + GeoGrid.Bucket cell = buckets.get(i); + String geohash = cell.getKeyAsString(); + + long bucketCount = cell.getDocCount(); + int expectedBucketCount = expectedDocCountsForGeoHash.get(geohash); + assertNotSame(bucketCount, 0); + assertEquals("Geohash " + geohash + " has wrong doc count ", + expectedBucketCount, bucketCount); + GeoPoint geoPoint = (GeoPoint) propertiesKeys[i]; + assertThat(stringEncode(geoPoint.lon(), geoPoint.lat(), precision), equalTo(geohash)); + assertThat(propertiesDocCounts[i], equalTo(bucketCount)); + } + } + public void testSimple() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 7657754b85e9e..fc727839b72f6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.apache.lucene.document.Document; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.DirectoryReader; @@ -28,10 +29,20 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.geo.CentroidCalculator; +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.GeoTestUtils; +import org.elasticsearch.common.geo.TriangleTreeReader; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.Point; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoBoundingBoxTests; -import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator; @@ -49,14 +60,13 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Function; +import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; import static org.hamcrest.Matchers.equalTo; public abstract class GeoGridAggregatorTestCase extends AggregatorTestCase { private static final String FIELD_NAME = "location"; - protected static final double GEOHASH_TOLERANCE = 1E-5D; /** * Generate a random precision according to the rules of the given aggregation. @@ -68,6 +78,16 @@ public abstract class GeoGridAggregatorTestCase */ protected abstract String hashAsString(double lng, double lat, int precision); + /** + * Return a point within the bounds of the tile grid + */ + protected abstract Point randomPoint(); + + /** + * Return the bounding tile as a {@link Rectangle} for a given point + */ + protected abstract Rectangle getTile(double lng, double lat, int precision); + /** * Create a new named {@link GeoGridAggregationBuilder}-derived builder */ @@ -84,148 +104,304 @@ protected List getSupportedValuesSourceTypes() { } public void testNoDocs() throws IOException { - testCase(new MatchAllDocsQuery(), FIELD_NAME, randomPrecision(), null, geoGrid -> { + testCase(new MatchAllDocsQuery(), FIELD_NAME, randomPrecision(), null, iw -> { + // Intentionally not writing any docs + }, geoGrid -> { assertEquals(0, geoGrid.getBuckets().size()); - }, iw -> { + }, new GeoShapeFieldMapper.GeoShapeFieldType()); + + testCase(new MatchAllDocsQuery(), FIELD_NAME, randomPrecision(), null, iw -> { // Intentionally not writing any docs - }); + }, geoGrid -> { + assertEquals(0, geoGrid.getBuckets().size()); + }, new GeoPointFieldMapper.GeoPointFieldType()); } public void testUnmapped() throws IOException { - testCase(new MatchAllDocsQuery(), "wrong_field", randomPrecision(), null, geoGrid -> { + final MappedFieldType fieldType; + if (randomBoolean()) { + fieldType = new GeoPointFieldMapper.GeoPointFieldType(); + } else { + fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); + } + + testCase(new MatchAllDocsQuery(), "wrong_field", randomPrecision(), null, iw -> {}, geoGrid -> { + assertEquals(0, geoGrid.getBuckets().size()); + }, fieldType); + + testCase(new MatchAllDocsQuery(), "wrong_field", randomPrecision(), null, iw -> { + iw.addDocument(Collections.singleton( + new BinaryGeoShapeDocValuesField(FIELD_NAME, GeoTestUtils.toDecodedTriangles(new Point(10D, 10D)), + new CentroidCalculator(new Point(10D, 10D))))); + }, geoGrid -> { assertEquals(0, geoGrid.getBuckets().size()); - }, iw -> { - iw.addDocument(Collections.singleton(new LatLonDocValuesField(FIELD_NAME, 10D, 10D))); - }); + }, fieldType); } - public void testUnmappedMissing() throws IOException { + public void testUnmappedMissingGeoPoint() throws IOException { GeoGridAggregationBuilder builder = createBuilder("_name") .field("wrong_field") .missing("53.69437,6.475031"); - testCase(new MatchAllDocsQuery(), randomPrecision(), null, geoGrid -> assertEquals(1, geoGrid.getBuckets().size()), - iw -> iw.addDocument(Collections.singleton(new LatLonDocValuesField(FIELD_NAME, 10D, 10D))), builder); + testCase(new MatchAllDocsQuery(), randomPrecision(), null, + iw -> iw.addDocument(Collections.singleton(new LatLonDocValuesField(FIELD_NAME, 10D, 10D))), + geoGrid -> assertEquals(1, geoGrid.getBuckets().size()), builder, new GeoPointFieldMapper.GeoPointFieldType()); + } + public void testUnmappedMissingGeoShape() throws IOException { + GeoGridAggregationBuilder builder = createBuilder("_name") + .field("wrong_field") + .missing("LINESTRING (30 10, 10 30, 40 40)"); + testCase(new MatchAllDocsQuery(), 1, null, + iw -> iw.addDocument(Collections.singleton(new LatLonDocValuesField(FIELD_NAME, 10D, 10D))), + geoGrid -> assertEquals(1, geoGrid.getBuckets().size()), builder, new GeoPointFieldMapper.GeoPointFieldType()); } - public void testWithSeveralDocs() throws IOException { + + public void testGeoPointWithSeveralDocs() throws IOException { int precision = randomPrecision(); int numPoints = randomIntBetween(8, 128); Map expectedCountPerGeoHash = new HashMap<>(); - testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, null, geoHashGrid -> { - assertEquals(expectedCountPerGeoHash.size(), geoHashGrid.getBuckets().size()); - for (GeoGrid.Bucket bucket : geoHashGrid.getBuckets()) { - assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); - } - assertTrue(AggregationInspectionHelper.hasValue(geoHashGrid)); - }, iw -> { - List points = new ArrayList<>(); - Set distinctHashesPerDoc = new HashSet<>(); - for (int pointId = 0; pointId < numPoints; pointId++) { - double lat = (180d * randomDouble()) - 90d; - double lng = (360d * randomDouble()) - 180d; + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, null, iw -> { + List points = new ArrayList<>(); + Set distinctHashesPerDoc = new HashSet<>(); + for (int pointId = 0; pointId < numPoints; pointId++) { + double lat = (180d * randomDouble()) - 90d; + double lng = (360d * randomDouble()) - 180d; - // Precision-adjust longitude/latitude to avoid wrong bucket placement - // Internally, lat/lng get converted to 32 bit integers, loosing some precision. - // This does not affect geohashing because geohash uses the same algorithm, - // but it does affect other bucketing algos, thus we need to do the same steps here. - lng = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lng)); - lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); + // Precision-adjust longitude/latitude to avoid wrong bucket placement + // Internally, lat/lng get converted to 32 bit integers, loosing some precision. + // This does not affect geohashing because geohash uses the same algorithm, + // but it does affect other bucketing algos, thus we need to do the same steps here. + lng = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lng)); + lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); - points.add(new LatLonDocValuesField(FIELD_NAME, lat, lng)); - String hash = hashAsString(lng, lat, precision); - if (distinctHashesPerDoc.contains(hash) == false) { - expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1); + points.add(new LatLonDocValuesField(FIELD_NAME, lat, lng)); + String hash = hashAsString(lng, lat, precision); + if (distinctHashesPerDoc.contains(hash) == false) { + expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1); + } + distinctHashesPerDoc.add(hash); + if (usually()) { + iw.addDocument(points); + points.clear(); + distinctHashesPerDoc.clear(); + } } - distinctHashesPerDoc.add(hash); - if (usually()) { + if (points.size() != 0) { iw.addDocument(points); - points.clear(); - distinctHashesPerDoc.clear(); } - } - if (points.size() != 0) { - iw.addDocument(points); - } - }); + }, + geoHashGrid -> { + assertEquals(expectedCountPerGeoHash.size(), geoHashGrid.getBuckets().size()); + for (GeoGrid.Bucket bucket : geoHashGrid.getBuckets()) { + assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); + } + assertTrue(AggregationInspectionHelper.hasValue(geoHashGrid)); + }, new GeoPointFieldMapper.GeoPointFieldType()); } - public void testBounds() throws IOException { - final int numDocs = randomIntBetween(64, 256); + public void testGeoPointBounds() throws IOException { + final int precision = randomPrecision(); + final int numDocs = randomIntBetween(100, 200); + int numDocsWithin = 0; final GeoGridAggregationBuilder builder = createBuilder("_name"); expectThrows(IllegalArgumentException.class, () -> builder.precision(-1)); expectThrows(IllegalArgumentException.class, () -> builder.precision(30)); - // only consider bounding boxes that are at least GEOHASH_TOLERANCE wide and have quantized coordinates - GeoBoundingBox bbox = randomValueOtherThanMany( - (b) -> Math.abs(GeoUtils.normalizeLon(b.right()) - GeoUtils.normalizeLon(b.left())) < GEOHASH_TOLERANCE, - GeoBoundingBoxTests::randomBBox); - Function encodeDecodeLat = (lat) -> GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); - Function encodeDecodeLon = (lon) -> GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); - bbox.topLeft().reset(encodeDecodeLat.apply(bbox.top()), encodeDecodeLon.apply(bbox.left())); - bbox.bottomRight().reset(encodeDecodeLat.apply(bbox.bottom()), encodeDecodeLon.apply(bbox.right())); + GeoBoundingBox bbox = GeoBoundingBoxTests.randomBBox(); + final double boundsTop = bbox.top(); + final double boundsBottom = bbox.bottom(); + final double boundsWestLeft; + final double boundsWestRight; + final double boundsEastLeft; + final double boundsEastRight; + final boolean crossesDateline; + if (bbox.right() < bbox.left()) { + boundsWestLeft = -180; + boundsWestRight = bbox.right(); + boundsEastLeft = bbox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = bbox.left(); + boundsEastRight = bbox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } - int in = 0, out = 0; List docs = new ArrayList<>(); - while (in + out < numDocs) { - if (bbox.left() > bbox.right()) { - if (randomBoolean()) { - double lonWithin = randomBoolean() ? - randomDoubleBetween(bbox.left(), 180.0, true) - : randomDoubleBetween(-180.0, bbox.right(), true); - double latWithin = randomDoubleBetween(bbox.bottom(), bbox.top(), true); - in++; - docs.add(new LatLonDocValuesField(FIELD_NAME, latWithin, lonWithin)); - } else { - double lonOutside = randomDoubleBetween(bbox.left(), bbox.right(), true); - double latOutside = randomDoubleBetween(bbox.top(), -90, false); - out++; - docs.add(new LatLonDocValuesField(FIELD_NAME, latOutside, lonOutside)); + for (int i = 0; i < numDocs; i++) { + Point p; + p = randomPoint(); + double x = GeoTestUtils.encodeDecodeLon(p.getX()); + double y = GeoTestUtils.encodeDecodeLat(p.getY()); + Rectangle pointTile = getTile(x, y, precision); + + boolean intersectsBounds = boundsTop >= pointTile.getMinY() && boundsBottom <= pointTile.getMaxY() + && (boundsEastLeft <= pointTile.getMaxX() && boundsEastRight >= pointTile.getMinX() + || (crossesDateline && boundsWestLeft <= pointTile.getMaxX() && boundsWestRight >= pointTile.getMinX())); + if (intersectsBounds) { + numDocsWithin += 1; + } + docs.add(new LatLonDocValuesField(FIELD_NAME, p.getLat(), p.getLon())); + } + + final long numDocsInBucket = numDocsWithin; + + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, bbox, iw -> { + for (LatLonDocValuesField docField : docs) { + iw.addDocument(Collections.singletonList(docField)); } - } else { - if (randomBoolean()) { - double lonWithin = randomDoubleBetween(bbox.left(), bbox.right(), true); - double latWithin = randomDoubleBetween(bbox.bottom(), bbox.top(), true); - in++; - docs.add(new LatLonDocValuesField(FIELD_NAME, latWithin, lonWithin)); - } else { - double lonOutside = GeoUtils.normalizeLon(randomDoubleBetween(bbox.right(), 180.001, false)); - double latOutside = GeoUtils.normalizeLat(randomDoubleBetween(bbox.top(), 90.001, false)); - out++; - docs.add(new LatLonDocValuesField(FIELD_NAME, latOutside, lonOutside)); + }, + geoGrid -> { + assertTrue(AggregationInspectionHelper.hasValue(geoGrid)); + long docCount = 0; + for (int i = 0; i < geoGrid.getBuckets().size(); i++) { + docCount += geoGrid.getBuckets().get(i).getDocCount(); } + assertThat(docCount, equalTo(numDocsInBucket)); + }, new GeoPointFieldMapper.GeoPointFieldType()); + } + + public void testGeoShapeBounds() throws IOException { + final int precision = randomPrecision(); + final int numDocs = randomIntBetween(100, 200); + int numDocsWithin = 0; + final GeoGridAggregationBuilder builder = createBuilder("_name"); + + expectThrows(IllegalArgumentException.class, () -> builder.precision(-1)); + expectThrows(IllegalArgumentException.class, () -> builder.precision(30)); + + GeoBoundingBox bbox = GeoBoundingBoxTests.randomBBox(); + final double boundsTop = bbox.top(); + final double boundsBottom = bbox.bottom(); + final double boundsWestLeft; + final double boundsWestRight; + final double boundsEastLeft; + final double boundsEastRight; + final boolean crossesDateline; + if (bbox.right() < bbox.left()) { + boundsWestLeft = -180; + boundsWestRight = bbox.right(); + boundsEastLeft = bbox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = bbox.left(); + boundsEastRight = bbox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + + List docs = new ArrayList<>(); + List points = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + Point p; + p = randomPoint(); + double x = GeoTestUtils.encodeDecodeLon(p.getX()); + double y = GeoTestUtils.encodeDecodeLat(p.getY()); + Rectangle pointTile = getTile(x, y, precision); + + + TriangleTreeReader reader = triangleTreeReader(p, GeoShapeCoordinateEncoder.INSTANCE); + GeoRelation tileRelation = reader.relateTile(GeoShapeCoordinateEncoder.INSTANCE.encodeX(pointTile.getMinX()), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(pointTile.getMinY()), + GeoShapeCoordinateEncoder.INSTANCE.encodeX(pointTile.getMaxX()), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(pointTile.getMaxY())); + boolean intersectsBounds = boundsTop >= pointTile.getMinY() && boundsBottom <= pointTile.getMaxY() + && (boundsEastLeft <= pointTile.getMaxX() && boundsEastRight >= pointTile.getMinX() + || (crossesDateline && boundsWestLeft <= pointTile.getMaxX() && boundsWestRight >= pointTile.getMinX())); + if (tileRelation != GeoRelation.QUERY_DISJOINT && intersectsBounds) { + numDocsWithin += 1; } + + points.add(p); + docs.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(p), new CentroidCalculator(p))); } - final long numDocsInBucket = in; - final int precision = randomPrecision(); + final long numDocsInBucket = numDocsWithin; + + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, bbox, iw -> { + for (BinaryGeoShapeDocValuesField docField : docs) { + iw.addDocument(Collections.singletonList(docField)); + } + }, + geoGrid -> { + assertThat(AggregationInspectionHelper.hasValue(geoGrid), equalTo(numDocsInBucket > 0)); + long docCount = 0; + for (int i = 0; i < geoGrid.getBuckets().size(); i++) { + docCount += geoGrid.getBuckets().get(i).getDocCount(); + } + assertThat(docCount, equalTo(numDocsInBucket)); + }, new GeoShapeFieldMapper.GeoShapeFieldType()); + } + + public void testGeoShapeWithSeveralDocs() throws IOException { + int precision = randomIntBetween(1, 4); + int numShapes = randomIntBetween(8, 128); + Map expectedCountPerGeoHash = new HashMap<>(); + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, null, iw -> { + List shapes = new ArrayList<>(); + Document document = new Document(); + Set distinctHashesPerDoc = new HashSet<>(); + for (int shapeId = 0; shapeId < numShapes; shapeId++) { + // undefined close to pole + double lat = (170.10225756d * randomDouble()) - 85.05112878d; + double lng = (360d * randomDouble()) - 180d; + + // Precision-adjust longitude/latitude to avoid wrong bucket placement + // Internally, lat/lng get converted to 32 bit integers, loosing some precision. + // This does not affect geohashing because geohash uses the same algorithm, + // but it does affect other bucketing algos, thus we need to do the same steps here. + lng = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lng)); + lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); - testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, bbox, geoGrid -> { - assertTrue(AggregationInspectionHelper.hasValue(geoGrid)); - long docCount = 0; - for (int i = 0; i < geoGrid.getBuckets().size(); i++) { - docCount += geoGrid.getBuckets().get(i).getDocCount(); + shapes.add(new Point(lng, lat)); + String hash = hashAsString(lng, lat, precision); + if (distinctHashesPerDoc.contains(hash) == false) { + expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1); + } + distinctHashesPerDoc.add(hash); + if (usually()) { + Geometry geometry = new MultiPoint(new ArrayList<>(shapes)); + document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(geometry), new CentroidCalculator(geometry))); + iw.addDocument(document); + shapes.clear(); + distinctHashesPerDoc.clear(); + document.clear(); + } + } + if (shapes.size() != 0) { + Geometry geometry = new MultiPoint(new ArrayList<>(shapes)); + document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(geometry), new CentroidCalculator(geometry))); + iw.addDocument(document); } - assertThat(docCount, equalTo(numDocsInBucket)); - }, iw -> { - for (LatLonDocValuesField docField : docs) { - iw.addDocument(Collections.singletonList(docField)); + }, geoHashGrid -> { + assertEquals(expectedCountPerGeoHash.size(), geoHashGrid.getBuckets().size()); + for (GeoGrid.Bucket bucket : geoHashGrid.getBuckets()) { + assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); } - }); + assertTrue(AggregationInspectionHelper.hasValue(geoHashGrid)); + }, new GeoShapeFieldMapper.GeoShapeFieldType()); } private void testCase(Query query, String field, int precision, GeoBoundingBox geoBoundingBox, - Consumer> verify, - CheckedConsumer buildIndex) throws IOException { - testCase(query, precision, geoBoundingBox, verify, buildIndex, createBuilder("_name").field(field)); + CheckedConsumer buildIndex, + Consumer> verify, MappedFieldType fieldType) throws IOException { + testCase(query, precision, geoBoundingBox, buildIndex, verify, createBuilder("_name").field(field), fieldType); } + @SuppressWarnings("unchecked") private void testCase(Query query, int precision, GeoBoundingBox geoBoundingBox, - Consumer> verify, - CheckedConsumer buildIndex, - GeoGridAggregationBuilder aggregationBuilder) throws IOException { + CheckedConsumer buildIndex, Consumer> verify, + GeoGridAggregationBuilder aggregationBuilder, MappedFieldType fieldType) throws IOException { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); buildIndex.accept(indexWriter); @@ -234,16 +410,15 @@ private void testCase(Query query, int precision, GeoBoundingBox geoBoundingBox, IndexReader indexReader = DirectoryReader.open(directory); IndexSearcher indexSearcher = newSearcher(indexReader, true, true); + fieldType.setHasDocValues(true); + fieldType.setName(FIELD_NAME); + aggregationBuilder.precision(precision); if (geoBoundingBox != null) { aggregationBuilder.setGeoBoundingBox(geoBoundingBox); assertThat(aggregationBuilder.geoBoundingBox(), equalTo(geoBoundingBox)); } - MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType(); - fieldType.setHasDocValues(true); - fieldType.setName(FIELD_NAME); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); indexSearcher.search(query, aggregator); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java new file mode 100644 index 0000000000000..9c64dccfafff9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -0,0 +1,517 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoBoundingBoxTests; +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.TriangleTreeReader; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.common.geo.GeoTestUtils.encodeDecodeLat; +import static org.elasticsearch.common.geo.GeoTestUtils.encodeDecodeLon; +import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.LATITUDE_MASK; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_LATITUDE_MASK; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK; + +import static org.hamcrest.Matchers.equalTo; + +public class GeoGridTilerTests extends ESTestCase { + private static final GeoTileGridTiler GEOTILE = new GeoTileGridTiler(); + private static final GeoHashGridTiler GEOHASH = new GeoHashGridTiler(); + + public void testGeoTile() throws Exception { + double x = randomDouble(); + double y = randomDouble(); + int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + assertThat(GEOTILE.encode(x, y, precision), equalTo(GeoTileUtils.longEncode(x, y, precision))); + + // create rectangle within tile and check bound counts + Rectangle tile = GeoTileUtils.toBoundingBox(1309, 3166, 13); + Rectangle shapeRectangle = new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, + tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001); + TriangleTreeReader reader = triangleTreeReader(shapeRectangle, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + + // test shape within tile bounds + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + int count = GEOTILE.setValues(values, value, 13); + assertThat(count, equalTo(1)); + } + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + int count = GEOTILE.setValues(values, value, 14); + assertThat(count, equalTo(4)); + } + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + int count = GEOTILE.setValues(values, value, 15); + assertThat(count, equalTo(16)); + } + } + + public void testAdvancePointValue() { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(1, 6); + int size = randomIntBetween(1, 10); + long[] values = new long[size]; + int idx = randomIntBetween(0, size - 1); + Point point = GeometryTestUtils.randomPoint(false); + for (GeoGridTiler tiler : List.of(GEOTILE, GEOHASH)) { + int newIdx = tiler.advancePointValue(values, point.getX(), point.getY(), precision, idx); + assertThat(newIdx, equalTo(idx + 1)); + assertThat(values[idx], equalTo(tiler.encode(point.getX(), point.getY(), precision))); + } + } + } + + public void testBoundedGeotileAdvancePointValue() { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(1, 6); + int size = randomIntBetween(1, 10); + long[] values = new long[size]; + int idx = randomIntBetween(0, size - 1); + Point point = GeometryTestUtils.randomPoint(false); + GeoBoundingBox geoBoundingBox = GeoBoundingBoxTests.randomBBox(); + + BoundedGeoTileGridTiler tiler = new BoundedGeoTileGridTiler(geoBoundingBox); + int newIdx = tiler.advancePointValue(values, point.getX(), point.getY(), precision, idx); + if (newIdx == idx + 1) { + assertTrue(tiler.cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(values[idx]))); + assertThat(values[idx], equalTo(tiler.encode(point.getX(), point.getY(), precision))); + assertThat(newIdx, equalTo(idx + 1)); + } else { + assertThat(newIdx, equalTo(idx)); + assertThat(values[idx], equalTo(0L)); + } + } + } + + public void testBoundedGeohashAdvancePointValue() { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(1, 6); + int size = randomIntBetween(1, 10); + long[] values = new long[size]; + int idx = randomIntBetween(0, size - 1); + Point point = GeometryTestUtils.randomPoint(false); + GeoBoundingBox geoBoundingBox = GeoBoundingBoxTests.randomBBox(); + + BoundedGeoHashGridTiler tiler = new BoundedGeoHashGridTiler(geoBoundingBox); + int newIdx = tiler.advancePointValue(values, point.getX(), point.getY(), precision, idx); + if (newIdx == idx + 1) { + assertTrue(tiler.cellIntersectsGeoBoundingBox(Geohash.toBoundingBox(Geohash.stringEncode(values[idx])))); + assertThat(values[idx], equalTo(tiler.encode(point.getX(), point.getY(), precision))); + assertThat(newIdx, equalTo(idx + 1)); + } else { + assertThat(newIdx, equalTo(idx)); + assertThat(values[idx], equalTo(0L)); + } + } + } + + public void testGeoTileSetValuesBruteAndRecursiveMultiline() throws Exception { + MultiLine geometry = GeometryTestUtils.randomMultiLine(false); + checkGeoTileSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); + } + + public void testGeoTileSetValuesBruteAndRecursivePolygon() throws Exception { + Geometry geometry = GeometryTestUtils.randomPolygon(false); + checkGeoTileSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); + } + + public void testGeoTileSetValuesBruteAndRecursivePoints() throws Exception { + Geometry geometry = randomBoolean() ? GeometryTestUtils.randomPoint(false) : GeometryTestUtils.randomMultiPoint(false); + checkGeoTileSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); + } + + // tests that bounding boxes of shapes crossing the dateline are correctly wrapped + public void testGeoTileSetValuesBoundingBoxes_BoundedGeoShapeCellValues() throws Exception { + for (int i = 0; i < 1; i++) { + int precision = randomIntBetween(0, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry geometry = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { + try { + indexer.prepareForIndexing(g); + return false; + } catch (Exception e) { + return true; + } + }, () -> boxToGeo(GeoBoundingBoxTests.randomBBox()))); + + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + GeoBoundingBox geoBoundingBox = GeoBoundingBoxTests.randomBBox(); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues cellValues = new GeoShapeCellValues(null, precision, GEOTILE); + + int numTiles = new BoundedGeoTileGridTiler(geoBoundingBox).setValues(cellValues, value, precision); + int expected = numTiles(value, precision, geoBoundingBox); + + assertThat(numTiles, equalTo(expected)); + } + } + + // test random rectangles that can cross the date-line and verify that there are an expected + // number of tiles returned + public void testGeoTileSetValuesBoundingBoxes_UnboundedGeoShapeCellValues() throws Exception { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(0, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry geometry = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { + try { + indexer.prepareForIndexing(g); + return false; + } catch (Exception e) { + return true; + } + }, () -> boxToGeo(GeoBoundingBoxTests.randomBBox()))); + + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + CellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); + int expected = numTiles(value, precision); + assertThat(numTiles, equalTo(expected)); + } + } + + public void testTilerMatchPoint() throws Exception { + int precision = randomIntBetween(0, 4); + Point originalPoint = GeometryTestUtils.randomPoint(false); + int xTile = GeoTileUtils.getXTile(originalPoint.getX(), 1 << precision); + int yTile = GeoTileUtils.getYTile(originalPoint.getY(), 1 << precision); + Rectangle bbox = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + + Point[] pointCorners = new Point[] { + // tile corners + new Point(bbox.getMinX(), bbox.getMinY()), + new Point(bbox.getMinX(), bbox.getMaxY()), + new Point(bbox.getMaxX(), bbox.getMinY()), + new Point(bbox.getMaxX(), bbox.getMaxY()), + // tile edge midpoints + new Point(bbox.getMinX(), (bbox.getMinY() + bbox.getMaxY()) / 2), + new Point(bbox.getMaxX(), (bbox.getMinY() + bbox.getMaxY()) / 2), + new Point((bbox.getMinX() + bbox.getMaxX()) / 2, bbox.getMinY()), + new Point((bbox.getMinX() + bbox.getMaxX()) / 2, bbox.getMaxY()), + }; + + for (Point point : pointCorners) { + if (point.getX() == GeoUtils.MAX_LON || point.getY() == -LATITUDE_MASK) { + continue; + } + TriangleTreeReader reader = triangleTreeReader(point, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); + assertThat(numTiles, equalTo(1)); + long tilerHash = unboundedCellValues.getValues()[0]; + long pointHash = GeoTileUtils.longEncode(encodeDecodeLon(point.getX()), encodeDecodeLat(point.getY()), precision); + assertThat(tilerHash, equalTo(pointHash)); + } + } + + public void testGeoHash() throws Exception { + double x = randomDouble(); + double y = randomDouble(); + int precision = randomIntBetween(0, 6); + assertThat(GEOHASH.encode(x, y, precision), equalTo(Geohash.longEncode(x, y, precision))); + + Rectangle tile = Geohash.toBoundingBox(Geohash.stringEncode(x, y, 5)); + + Rectangle shapeRectangle = new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, + tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001); + TriangleTreeReader reader = triangleTreeReader(shapeRectangle, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + + // test shape within tile bounds + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + int count = GEOHASH.setValues(values, value, 5); + assertThat(count, equalTo(1)); + } + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + int count = GEOHASH.setValues(values, value, 6); + assertThat(count, equalTo(32)); + } + { + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); + int count = GEOHASH.setValues(values, value, 7); + assertThat(count, equalTo(1024)); + } + } + + private boolean tileIntersectsBounds(int x, int y, int precision, GeoBoundingBox bounds) { + if (bounds == null) { + return true; + } + final double boundsWestLeft; + final double boundsWestRight; + final double boundsEastLeft; + final double boundsEastRight; + final boolean crossesDateline; + if (bounds.right() < bounds.left()) { + boundsWestLeft = -180; + boundsWestRight = bounds.right(); + boundsEastLeft = bounds.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { + boundsEastLeft = bounds.left(); + boundsEastRight = bounds.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + + Rectangle tile = GeoTileUtils.toBoundingBox(x, y, precision); + + return (bounds.top() >= tile.getMinY() && bounds.bottom() <= tile.getMaxY() + && (boundsEastLeft <= tile.getMaxX() && boundsEastRight >= tile.getMinX() + || (crossesDateline && boundsWestLeft <= tile.getMaxX() && boundsWestRight >= tile.getMinX()))); + } + + private int numTiles(MultiGeoValues.GeoValue geoValue, int precision, GeoBoundingBox geoBox) throws Exception { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + int count = 0; + + if (precision == 0) { + return 1; + } else if ((bounds.top > LATITUDE_MASK && bounds.bottom > LATITUDE_MASK) + || (bounds.top < -LATITUDE_MASK && bounds.bottom < -LATITUDE_MASK)) { + return 0; + } + final double tiles = 1 << precision; + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + if ((bounds.posLeft >= 0 && bounds.posRight >= 0) && (bounds.negLeft < 0 && bounds.negRight < 0)) { + // box one + int minXTileNeg = GeoTileUtils.getXTile(bounds.negLeft, (long) tiles); + int maxXTileNeg = GeoTileUtils.getXTile(bounds.negRight, (long) tiles); + + for (int x = minXTileNeg; x <= maxXTileNeg; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + + // box two + int minXTilePos = GeoTileUtils.getXTile(bounds.posLeft, (long) tiles); + if (minXTilePos > maxXTileNeg + 1) { + minXTilePos -= 1; + } + + int maxXTilePos = GeoTileUtils.getXTile(bounds.posRight, (long) tiles); + + for (int x = minXTilePos; x <= maxXTilePos; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } else { + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + + if (minXTile == maxXTile && minYTile == maxYTile) { + return tileIntersectsBounds(minXTile, minYTile, precision, geoBox) ? 1 : 0; + } + + for (int x = minXTile; x <= maxXTile; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } + } + + private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Exception { + int precision = randomIntBetween(1, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOTILE); + int recursiveCount; + { + recursiveCount = GEOTILE.setValuesByRasterization(0, 0, 0, recursiveValues, 0, precision, value); + } + GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOTILE); + int bruteForceCount; + { + final double tiles = 1 << precision; + MultiGeoValues.BoundingBox bounds = value.boundingBox(); + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision, minXTile, minYTile, maxXTile, maxYTile); + } + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); + long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); + Arrays.sort(recursive); + Arrays.sort(bruteForce); + assertArrayEquals(geometry.toString(), recursive, bruteForce); + } + + private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Exception { + int precision = randomIntBetween(1, 3); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOHASH); + int recursiveCount; + { + recursiveCount = GEOHASH.setValuesByRasterization("", recursiveValues, 0, precision, value); + } + GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOHASH); + int bruteForceCount; + { + MultiGeoValues.BoundingBox bounds = value.boundingBox(); + bruteForceCount = GEOHASH.setValuesByBruteForceScan(bruteForceValues, value, precision, bounds); + } + + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); + long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); + Arrays.sort(recursive); + Arrays.sort(bruteForce); + assertArrayEquals(geometry.toString(), recursive, bruteForce); + } + + + static Geometry boxToGeo(GeoBoundingBox geoBox) { + // turn into polygon + if (geoBox.right() < geoBox.left() && geoBox.right() != -180) { + return new MultiPolygon(List.of( + new Polygon(new LinearRing( + new double[] { -180, geoBox.right(), geoBox.right(), -180, -180 }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })), + new Polygon(new LinearRing( + new double[] { geoBox.left(), 180, 180, geoBox.left(), geoBox.left() }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })) + )); + } else { + double right = GeoUtils.normalizeLon(geoBox.right()); + return new Polygon(new LinearRing( + new double[] { geoBox.left(), right, right, geoBox.left(), geoBox.left() }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })); + } + } + + private int numTiles(MultiGeoValues.GeoValue geoValue, int precision) { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + int count = 0; + + if (precision == 0) { + return 1; + } + + if ((bounds.top > NORMALIZED_LATITUDE_MASK && bounds.bottom > NORMALIZED_LATITUDE_MASK) + || (bounds.top < NORMALIZED_NEGATIVE_LATITUDE_MASK && bounds.bottom < NORMALIZED_NEGATIVE_LATITUDE_MASK)) { + return 0; + } + + final double tiles = 1 << precision; + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + if ((bounds.posLeft >= 0 && bounds.posRight >= 0) && (bounds.negLeft < 0 && bounds.negRight < 0)) { + // box one + int minXTileNeg = GeoTileUtils.getXTile(bounds.negLeft, (long) tiles); + int maxXTileNeg = GeoTileUtils.getXTile(bounds.negRight, (long) tiles); + + for (int x = minXTileNeg; x <= maxXTileNeg; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + + // box two + int minXTilePos = GeoTileUtils.getXTile(bounds.posLeft, (long) tiles); + if (minXTilePos > maxXTileNeg + 1) { + minXTilePos -= 1; + } + + int maxXTilePos = GeoTileUtils.getXTile(bounds.posRight, (long) tiles); + + for (int x = minXTilePos; x <= maxXTilePos; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } else { + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + + if (minXTile == maxXTile && minYTile == maxYTile) { + return 1; + } + + for (int x = minXTile; x <= maxXTile; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java index 7fa517807f619..2aab993c1d8c0 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java @@ -19,6 +19,11 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; + import static org.elasticsearch.geometry.utils.Geohash.stringEncode; public class GeoHashGridAggregatorTests extends GeoGridAggregatorTestCase { @@ -33,6 +38,16 @@ protected String hashAsString(double lng, double lat, int precision) { return stringEncode(lng, lat, precision); } + @Override + protected Point randomPoint() { + return GeometryTestUtils.randomPoint(false); + } + + @Override + protected Rectangle getTile(double lng, double lat, int precision) { + return Geohash.toBoundingBox(Geohash.stringEncode(lng, lat, precision)); + } + @Override protected GeoGridAggregationBuilder createBuilder(String name) { return new GeoHashGridAggregationBuilder(name); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index 85b2306403230..fd4c52a0f7d8d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -19,6 +19,10 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; + public class GeoTileGridAggregatorTests extends GeoGridAggregatorTestCase { @Override @@ -31,6 +35,17 @@ protected String hashAsString(double lng, double lat, int precision) { return GeoTileUtils.stringEncode(GeoTileUtils.longEncode(lng, lat, precision)); } + @Override + protected Point randomPoint() { + return new Point(randomDoubleBetween(GeoUtils.MIN_LON, GeoUtils.MAX_LON, true), + randomDoubleBetween(-GeoTileUtils.LATITUDE_MASK, GeoTileUtils.LATITUDE_MASK, false)); + } + + @Override + protected Rectangle getTile(double lng, double lat, int precision) { + return GeoTileUtils.toBoundingBox(GeoTileUtils.longEncode(lng, lat, precision)); + } + @Override protected GeoGridAggregationBuilder createBuilder(String name) { return new GeoTileGridAggregationBuilder(name); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index fc5cf6cb910bd..cf3e8699b6894 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; @@ -28,8 +29,10 @@ import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.keyToGeoPoint; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class GeoTileUtilsTests extends ESTestCase { @@ -219,8 +222,8 @@ public void testGeoTileAsLongRoutines() { * so ensure they are clipped correctly. */ public void testSingularityAtPoles() { - double minLat = -85.05112878; - double maxLat = 85.05112878; + double minLat = -GeoTileUtils.LATITUDE_MASK; + double maxLat = GeoTileUtils.LATITUDE_MASK; double lon = randomIntBetween(-180, 180); double lat = randomBoolean() ? randomDoubleBetween(-90, minLat, true) @@ -231,4 +234,23 @@ public void testSingularityAtPoles() { String clippedTileIndex = stringEncode(longEncode(lon, clippedLat, zoom)); assertEquals(tileIndex, clippedTileIndex); } + + public void testPointToTile() { + int zoom = randomIntBetween(0, MAX_ZOOM); + int tiles = 1 << zoom; + int xTile = randomIntBetween(0, zoom); + int yTile = randomIntBetween(0, zoom); + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, zoom); + // check corners + assertThat(GeoTileUtils.getXTile(rectangle.getMinX(), tiles), equalTo(xTile)); + assertThat(GeoTileUtils.getXTile(rectangle.getMaxX(), tiles), equalTo(Math.min(tiles - 1, xTile + 1))); + assertThat(GeoTileUtils.getYTile(rectangle.getMaxY(), tiles), anyOf(equalTo(yTile - 1), equalTo(yTile))); + assertThat(GeoTileUtils.getYTile(rectangle.getMinY(), tiles), anyOf(equalTo(yTile + 1), equalTo(yTile))); + // check point inside + double x = randomDoubleBetween(rectangle.getMinX(), rectangle.getMaxX(), false); + double y = randomDoubleBetween(rectangle.getMinY(), rectangle.getMaxY(), false); + assertThat(GeoTileUtils.getXTile(x, tiles), equalTo(xTile)); + assertThat(GeoTileUtils.getYTile(y, tiles), equalTo(yTile)); + + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java index e9bef98f1f05f..93d125f317f8d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -23,21 +23,26 @@ import com.carrotsearch.hppc.ObjectIntMap; import com.carrotsearch.hppc.ObjectObjectHashMap; import com.carrotsearch.hppc.ObjectObjectMap; - +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.geo.RandomGeoGenerator; import java.util.ArrayList; @@ -51,7 +56,8 @@ @ESIntegTestCase.SuiteScopeTestCase public abstract class AbstractGeoTestCase extends ESIntegTestCase { - protected static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; + protected static final String SINGLE_VALUED_GEOPOINT_FIELD_NAME = "geopoint_value"; + protected static final String SINGLE_VALUED_GEOSHAPE_FIELD_NAME = "geoshape_value"; protected static final String MULTI_VALUED_FIELD_NAME = "geo_values"; protected static final String NUMBER_FIELD_NAME = "l_values"; protected static final String UNMAPPED_IDX_NAME = "idx_unmapped"; @@ -60,23 +66,33 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase { protected static final String DATELINE_IDX_NAME = "dateline_idx"; protected static final String HIGH_CARD_IDX_NAME = "high_card_idx"; protected static final String IDX_ZERO_NAME = "idx_zero"; + protected static final String IDX_NAME_7x = "idx_7x"; protected static int numDocs; protected static int numUniqueGeoPoints; protected static GeoPoint[] singleValues, multiValues; protected static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, - singleCentroid, multiCentroid, unmappedCentroid; + singleCentroid, singleShapeCentroid, multiCentroid, unmappedCentroid; protected static ObjectIntMap expectedDocCountsForGeoHash = null; protected static ObjectObjectMap expectedCentroidsForGeoHash = null; protected static final double GEOHASH_TOLERANCE = 1E-5D; + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex(UNMAPPED_IDX_NAME); assertAcked(prepareCreate(IDX_NAME) - .setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point", + .setMapping(SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point", SINGLE_VALUED_GEOSHAPE_FIELD_NAME, "type=geo_shape", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=keyword")); + assertAcked(prepareCreate(IDX_NAME_7x) + .setSettings(settings(VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0))) + .setMapping(SINGLE_VALUED_GEOSHAPE_FIELD_NAME, "type=geo_shape")); + singleTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); singleBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); @@ -115,9 +131,16 @@ public void setupSuiteScopeCluster() throws Exception { singleVal = singleValues[i % numUniqueGeoPoints]; multiVal[0] = multiValues[i % numUniqueGeoPoints]; multiVal[1] = multiValues[(i+1) % numUniqueGeoPoints]; + String singleValWKT = "POINT(" + singleVal.lon() + " " + singleVal.lat() + ")"; + builders.add(client().prepareIndex(IDX_NAME_7x).setSource(jsonBuilder() + .startObject() + .field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME, singleValWKT) + .endObject() + )); builders.add(client().prepareIndex(IDX_NAME).setSource(jsonBuilder() .startObject() - .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME, singleValWKT) .startArray(MULTI_VALUED_FIELD_NAME) .startArray().value(multiVal[0].lon()).value(multiVal[0].lat()).endArray() .startArray().value(multiVal[1].lon()).value(multiVal[1].lat()).endArray() @@ -133,41 +156,44 @@ public void setupSuiteScopeCluster() throws Exception { multiCentroid.lon() + (newMVLon - multiCentroid.lon()) / (i+1)); } - assertAcked(prepareCreate(EMPTY_IDX_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + assertAcked(prepareCreate(EMPTY_IDX_NAME).setMapping(SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point")); assertAcked(prepareCreate(DATELINE_IDX_NAME) - .setMapping(SINGLE_VALUED_FIELD_NAME, - "type=geo_point", MULTI_VALUED_FIELD_NAME, - "type=geo_point", NUMBER_FIELD_NAME, - "type=long", "tag", "type=keyword")); - - GeoPoint[] geoValues = new GeoPoint[5]; - geoValues[0] = new GeoPoint(38, 178); - geoValues[1] = new GeoPoint(12, -179); - geoValues[2] = new GeoPoint(-24, 170); - geoValues[3] = new GeoPoint(32, -175); - geoValues[4] = new GeoPoint(-11, 178); + .setMapping(SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point", + SINGLE_VALUED_GEOSHAPE_FIELD_NAME, "type=geo_shape", + MULTI_VALUED_FIELD_NAME, "type=geo_point", + NUMBER_FIELD_NAME, "type=long", "tag", "type=keyword")); + GeoPoint[] geoPointValues = new GeoPoint[5]; + geoPointValues[0] = new GeoPoint(38, 178); + geoPointValues[1] = new GeoPoint(12, -179); + geoPointValues[2] = new GeoPoint(-24, 170); + geoPointValues[3] = new GeoPoint(32, -175); + geoPointValues[4] = new GeoPoint(-11, 178); + Line line = new Line(new double[] { 178, -179, 170, -175, 178 }, new double[] { 38, 12, -24, 32, -11 }); + String lineAsWKT = new WellKnownText(false, new GeographyValidator(true)).toWKT(line); + CentroidCalculator centroidCalculator = new CentroidCalculator(line); + singleShapeCentroid = new GeoPoint(centroidCalculator.getY(), centroidCalculator.getX()); for (int i = 0; i < 5; i++) { builders.add(client().prepareIndex(DATELINE_IDX_NAME).setSource(jsonBuilder() .startObject() - .array(SINGLE_VALUED_FIELD_NAME, geoValues[i].lon(), geoValues[i].lat()) + .array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, geoPointValues[i].lon(), geoPointValues[i].lat()) + .field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME, lineAsWKT) .field(NUMBER_FIELD_NAME, i) .field("tag", "tag" + i) .endObject())); } assertAcked(prepareCreate(HIGH_CARD_IDX_NAME).setSettings(Settings.builder().put("number_of_shards", 2)) - .setMapping(SINGLE_VALUED_FIELD_NAME, - "type=geo_point", MULTI_VALUED_FIELD_NAME, - "type=geo_point", NUMBER_FIELD_NAME, - "type=long,store=true", + .setMapping(SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point", + MULTI_VALUED_FIELD_NAME, "type=geo_point", + NUMBER_FIELD_NAME, "type=long,store=true", "tag", "type=keyword")); for (int i = 0; i < 2000; i++) { singleVal = singleValues[i % numUniqueGeoPoints]; builders.add(client().prepareIndex(HIGH_CARD_IDX_NAME).setSource(jsonBuilder() .startObject() - .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, singleVal.lon(), singleVal.lat()) .startArray(MULTI_VALUED_FIELD_NAME) .startArray() .value(multiValues[i % numUniqueGeoPoints].lon()) @@ -185,8 +211,8 @@ public void setupSuiteScopeCluster() throws Exception { } builders.add(client().prepareIndex(IDX_ZERO_NAME).setSource( - jsonBuilder().startObject().array(SINGLE_VALUED_FIELD_NAME, 0.0, 1.0).endObject())); - assertAcked(prepareCreate(IDX_ZERO_NAME).setMapping(SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + jsonBuilder().startObject().array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, 0.0, 1.0).endObject())); + assertAcked(prepareCreate(IDX_ZERO_NAME).setMapping(SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point")); indexRandom(true, builders); ensureSearchable(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java index ef42a2d210eed..266878b8a637f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java @@ -27,11 +27,19 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; -import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.common.geo.GeoTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; +import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -40,6 +48,7 @@ import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.test.geo.RandomGeoGenerator; +import java.util.ArrayList; import java.util.List; import static org.elasticsearch.search.aggregations.metrics.InternalGeoBoundsTests.GEOHASH_TOLERANCE; @@ -48,7 +57,8 @@ import static org.hamcrest.Matchers.startsWith; public class GeoBoundsAggregatorTests extends AggregatorTestCase { - public void testEmpty() throws Exception { + + public void testEmptyGeoPoint() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg") .field("field") @@ -71,6 +81,30 @@ public void testEmpty() throws Exception { } } + public void testEmptyGeoShape() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg") + .field("field") + .wrapLongitude(false); + + MappedFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("field"); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertTrue(Double.isInfinite(bounds.top)); + assertTrue(Double.isInfinite(bounds.bottom)); + assertTrue(Double.isInfinite(bounds.posLeft)); + assertTrue(Double.isInfinite(bounds.posRight)); + assertTrue(Double.isInfinite(bounds.negLeft)); + assertTrue(Double.isInfinite(bounds.negRight)); + assertFalse(AggregationInspectionHelper.hasValue(bounds)); + } + } + } + public void testUnmappedFieldWithDocs() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { if (randomBoolean()) { @@ -84,7 +118,6 @@ public void testUnmappedFieldWithDocs() throws Exception { .wrapLongitude(false); MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType(); - fieldType.setHasDocValues(true); fieldType.setName("field"); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); @@ -158,7 +191,7 @@ public void testInvalidMissing() throws Exception { } } - public void testRandom() throws Exception { + public void testRandomPoints() throws Exception { double top = Double.NEGATIVE_INFINITY; double bottom = Double.POSITIVE_INFINITY; double posLeft = Double.POSITIVE_INFINITY; @@ -216,6 +249,212 @@ public void testRandom() throws Exception { } } + public void testRandomShapes() throws Exception { + double top = Double.NEGATIVE_INFINITY; + double bottom = Double.POSITIVE_INFINITY; + double posLeft = Double.POSITIVE_INFINITY; + double posRight = Double.NEGATIVE_INFINITY; + double negLeft = Double.POSITIVE_INFINITY; + double negRight = Double.NEGATIVE_INFINITY; + int numDocs = randomIntBetween(50, 100); + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + int numValues = randomIntBetween(1, 5); + List points = new ArrayList<>(); + for (int j = 0; j < numValues; j++) { + GeoPoint point = RandomGeoGenerator.randomPoint(random()); + points.add(new Point(point.getLon(), point.getLat())); + if (point.getLat() > top) { + top = point.getLat(); + } + if (point.getLat() < bottom) { + bottom = point.getLat(); + } + if (point.getLon() >= 0 && point.getLon() < posLeft) { + posLeft = point.getLon(); + } + if (point.getLon() >= 0 && point.getLon() > posRight) { + posRight = point.getLon(); + } + if (point.getLon() < 0 && point.getLon() < negLeft) { + negLeft = point.getLon(); + } + if (point.getLon() < 0 && point.getLon() > negRight) { + negRight = point.getLon(); + } + } + Geometry geometry = new MultiPoint(points); + doc.add(new BinaryGeoShapeDocValuesField("field", GeoTestUtils.toDecodedTriangles(geometry), + new CentroidCalculator(geometry))); + w.addDocument(doc); + } + GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg") + .field("field") + .wrapLongitude(false); + + MappedFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("field"); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertThat(bounds.top, closeTo(top, GEOHASH_TOLERANCE)); + assertThat(bounds.bottom, closeTo(bottom, GEOHASH_TOLERANCE)); + assertThat(bounds.posLeft, closeTo(posLeft, GEOHASH_TOLERANCE)); + assertThat(bounds.posRight, closeTo(posRight, GEOHASH_TOLERANCE)); + assertThat(bounds.negRight, closeTo(negRight, GEOHASH_TOLERANCE)); + assertThat(bounds.negLeft, closeTo(negLeft, GEOHASH_TOLERANCE)); + assertTrue(AggregationInspectionHelper.hasValue(bounds)); + } + } + } + + public void testFiji() throws Exception { + MultiPolygon geometryForIndexing = (MultiPolygon) GeoTestUtils.fromGeoJsonString("{\n" + + " \"type\": \"MultiPolygon\",\n" + + " \"coordinates\": [\n" + + " [\n" + + " [\n" + + " [\n" + + " 178.3736,\n" + + " -17.33992\n" + + " ],\n" + + " [\n" + + " 178.71806,\n" + + " -17.62846\n" + + " ],\n" + + " [\n" + + " 178.55271,\n" + + " -18.15059\n" + + " ],\n" + + " [\n" + + " 177.93266,\n" + + " -18.28799\n" + + " ],\n" + + " [\n" + + " 177.38146,\n" + + " -18.16432\n" + + " ],\n" + + " [\n" + + " 177.28504,\n" + + " -17.72465\n" + + " ],\n" + + " [\n" + + " 177.67087,\n" + + " -17.38114\n" + + " ],\n" + + " [\n" + + " 178.12557,\n" + + " -17.50481\n" + + " ],\n" + + " [\n" + + " 178.3736,\n" + + " -17.33992\n" + + " ]\n" + + " ]\n" + + " ],\n" + + " [\n" + + " [\n" + + " [\n" + + " 179.364143,\n" + + " -16.801354\n" + + " ],\n" + + " [\n" + + " 178.725059,\n" + + " -17.012042\n" + + " ],\n" + + " [\n" + + " 178.596839,\n" + + " -16.63915\n" + + " ],\n" + + " [\n" + + " 179.096609,\n" + + " -16.433984\n" + + " ],\n" + + " [\n" + + " 179.413509,\n" + + " -16.379054\n" + + " ],\n" + + " [\n" + + " 180,\n" + + " -16.067133\n" + + " ],\n" + + " [\n" + + " 180,\n" + + " -16.555217\n" + + " ],\n" + + " [\n" + + " 179.364143,\n" + + " -16.801354\n" + + " ]\n" + + " ]\n" + + " ],\n" + + " [\n" + + " [\n" + + " [\n" + + " -179.917369,\n" + + " -16.501783\n" + + " ],\n" + + " [\n" + + " -180,\n" + + " -16.555217\n" + + " ],\n" + + " [\n" + + " -180,\n" + + " -16.067133\n" + + " ],\n" + + " [\n" + + " -179.79332,\n" + + " -16.020882\n" + + " ],\n" + + " [\n" + + " -179.917369,\n" + + " -16.501783\n" + + " ]\n" + + " ]\n" + + " ]\n" + + " ]\n" + + " }"); + + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + Document doc = new Document(); + doc.add(new BinaryGeoShapeDocValuesField("fiji_shape", + GeoTestUtils.toDecodedTriangles(geometryForIndexing), new CentroidCalculator(geometryForIndexing))); + for (Polygon poly : geometryForIndexing) { + for (int i = 0; i < poly.getPolygon().length(); i++) { + doc.add(new LatLonDocValuesField("fiji_points", poly.getPolygon().getLat(i), poly.getPolygon().getLon(i))); + } + } + + w.addDocument(doc); + + boolean wrapLongitude = randomBoolean(); + GeoBoundsAggregationBuilder pointsAggBuilder = new GeoBoundsAggregationBuilder("my_agg") + .field("fiji_points").wrapLongitude(wrapLongitude); + MappedFieldType pointsType = new GeoPointFieldMapper.GeoPointFieldType(); + pointsType.setHasDocValues(true); + pointsType.setName("fiji_points"); + + GeoBoundsAggregationBuilder shapesAggBuilder = new GeoBoundsAggregationBuilder("my_agg") + .field("fiji_shape").wrapLongitude(wrapLongitude); + MappedFieldType shapeType = new GeoShapeFieldMapper.GeoShapeFieldType(); + shapeType.setHasDocValues(true); + shapeType.setName("fiji_shape"); + + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoBounds pointBounds = search(searcher, new MatchAllDocsQuery(), pointsAggBuilder, pointsType); + InternalGeoBounds shapeBounds = search(searcher, new MatchAllDocsQuery(), shapesAggBuilder, shapeType); + assertTrue(AggregationInspectionHelper.hasValue(pointBounds)); + assertTrue(AggregationInspectionHelper.hasValue(shapeBounds)); + assertThat(shapeBounds, equalTo(pointBounds)); + } + } + } + @Override protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { return new GeoBoundsAggregationBuilder("foo").field(fieldName); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java index baf9aab19dc82..6cb1033bddbaf 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java @@ -19,9 +19,12 @@ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.util.BigArray; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -34,30 +37,58 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.geoBounds; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; @ESIntegTestCase.SuiteScopeTestCase public class GeoBoundsIT extends AbstractGeoTestCase { - private static final String aggName = "geoBounds"; + private static final String geoPointAggName = "geoPointBounds"; + private static final String geoShapeAggName = "geoShapeBounds"; + + public void test7xIndexOnly() { + SearchPhaseExecutionException exception = expectThrows(SearchPhaseExecutionException.class, + () -> client().prepareSearch(IDX_NAME_7x) .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME)) + .get()); + assertNotNull(exception.getRootCause()); + assertThat(exception.getRootCause().getMessage(), + equalTo("Can't load fielddata on [geoshape_value] because fielddata is unsupported on fields of type [geo_shape]." + + " Use doc values instead.")); + } + + public void test7xIndexWith8Index() { + SearchResponse response = client().prepareSearch(IDX_NAME_7x, IDX_NAME) + .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME).wrapLongitude(false)).get(); + assertThat(response.status(), equalTo(RestStatus.OK)); + assertThat(response.getSuccessfulShards(), lessThan(response.getTotalShards())); + GeoBounds geoBounds = response.getAggregations().get(geoShapeAggName); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); + } public void testSingleValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) .wrapLongitude(false)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); @@ -71,7 +102,8 @@ public void testSingleValuedField_getProperty() throws Exception { .prepareSearch(IDX_NAME) .setQuery(matchAllQuery()) .addAggregation( - global("global").subAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false))) + global("global").subAggregation(geoBounds(geoPointAggName) + .field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(false))) .get(); assertSearchResponse(searchResponse); @@ -83,36 +115,38 @@ public void testSingleValuedField_getProperty() throws Exception { assertThat(global.getAggregations(), notNullValue()); assertThat(global.getAggregations().asMap().size(), equalTo(1)); - GeoBounds geobounds = global.getAggregations().get(aggName); + GeoBounds geobounds = global.getAggregations().get(geoPointAggName); assertThat(geobounds, notNullValue()); - assertThat(geobounds.getName(), equalTo(aggName)); - assertThat((GeoBounds) ((InternalAggregation)global).getProperty(aggName), sameInstance(geobounds)); + assertThat(geobounds.getName(), equalTo(geoPointAggName)); + assertThat((GeoBounds) ((InternalAggregation)global).getProperty(geoPointAggName), sameInstance(geobounds)); GeoPoint topLeft = geobounds.topLeft(); GeoPoint bottomRight = geobounds.bottomRight(); assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); - assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".top"), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); - assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".left"), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); - assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".bottom"), + assertThat((double) ((InternalAggregation)global).getProperty(geoPointAggName + ".top"), + closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat((double) ((InternalAggregation)global).getProperty(geoPointAggName + ".left"), + closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat((double) ((InternalAggregation)global).getProperty(geoPointAggName + ".bottom"), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); - assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".right"), + assertThat((double) ((InternalAggregation)global).getProperty(geoPointAggName + ".right"), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); } public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) - .addAggregation(geoBounds(aggName).field(MULTI_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(MULTI_VALUED_FIELD_NAME) .wrapLongitude(false)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(multiTopLeft.lat(), GEOHASH_TOLERANCE)); @@ -123,15 +157,15 @@ public void testMultiValuedField() throws Exception { public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch(UNMAPPED_IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) .wrapLongitude(false)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft, equalTo(null)); @@ -140,15 +174,15 @@ public void testUnmapped() throws Exception { public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME, UNMAPPED_IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) .wrapLongitude(false)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); @@ -160,23 +194,27 @@ public void testPartiallyUnmapped() throws Exception { public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch(EMPTY_IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) + .wrapLongitude(false)) + .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME) .wrapLongitude(false)) .get(); - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); - GeoBounds geoBounds = searchResponse.getAggregations().get(aggName); - assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); - GeoPoint topLeft = geoBounds.topLeft(); - GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft, equalTo(null)); - assertThat(bottomRight, equalTo(null)); + for (String aggName : List.of(geoPointAggName, geoShapeAggName)) { + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + GeoBounds geoBounds = searchResponse.getAggregations().get(aggName); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo(aggName)); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft, equalTo(null)); + assertThat(bottomRight, equalTo(null)); + } } public void testSingleValuedFieldNearDateLine() throws Exception { SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) .wrapLongitude(false)) .get(); @@ -185,9 +223,9 @@ public void testSingleValuedFieldNearDateLine() throws Exception { GeoPoint geoValuesTopLeft = new GeoPoint(38, -179); GeoPoint geoValuesBottomRight = new GeoPoint(-24, 178); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); @@ -197,24 +235,42 @@ public void testSingleValuedFieldNearDateLine() throws Exception { } public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { - - GeoPoint geoValuesTopLeft = new GeoPoint(38, 170); - GeoPoint geoValuesBottomRight = new GeoPoint(-24, -175); SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(true)) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(true)) + .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME).wrapLongitude(true)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); - assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); - GeoPoint topLeft = geoBounds.topLeft(); - GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); - assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); - assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); - assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); + // test geo_point + { + GeoPoint geoValuesTopLeft = new GeoPoint(38, 170); + GeoPoint geoValuesBottomRight = new GeoPoint(-24, -175); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); + } + + // test geo_shape, should not wrap dateline + { + GeoPoint geoValuesTopLeft = new GeoPoint(38, -179); + GeoPoint geoValuesBottomRight = new GeoPoint(-24, 178); + GeoBounds geoBounds = response.getAggregations().get(geoShapeAggName); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo(geoShapeAggName)); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); + } } /** @@ -222,8 +278,8 @@ public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { */ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) - .addAggregation(terms("terms").field(NUMBER_FIELD_NAME).subAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) - .wrapLongitude(false))) + .addAggregation(terms("terms").field(NUMBER_FIELD_NAME).subAggregation(geoBounds(geoPointAggName) + .field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(false))) .get(); assertSearchResponse(response); @@ -237,9 +293,9 @@ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); assertThat("InternalBucket " + bucket.getKey() + " has wrong number of documents", bucket.getDocCount(), equalTo(1L)); - GeoBounds geoBounds = bucket.getAggregations().get(aggName); + GeoBounds geoBounds = bucket.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); assertThat(geoBounds.topLeft().getLat(), allOf(greaterThanOrEqualTo(-90.0), lessThanOrEqualTo(90.0))); assertThat(geoBounds.topLeft().getLon(), allOf(greaterThanOrEqualTo(-180.0), lessThanOrEqualTo(180.0))); assertThat(geoBounds.bottomRight().getLat(), allOf(greaterThanOrEqualTo(-90.0), lessThanOrEqualTo(90.0))); @@ -249,13 +305,13 @@ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { public void testSingleValuedFieldWithZeroLon() throws Exception { SearchResponse response = client().prepareSearch(IDX_ZERO_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false)).get(); + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(false)).get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); @@ -263,4 +319,41 @@ public void testSingleValuedFieldWithZeroLon() throws Exception { assertThat(bottomRight.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); assertThat(bottomRight.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); } + + public void testIncorrectFieldType() { + SearchRequestBuilder searchWithKeywordField = client().prepareSearch(DATELINE_IDX_NAME) + .addAggregation(geoBounds("agg").field("tag")); + assertFailures(searchWithKeywordField, RestStatus.BAD_REQUEST, + containsString("Expected geo_point or geo_shape type on field [tag], but got [keyword]")); + + { + SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) + .addAggregation(geoBounds("agg").missing(randomBoolean() ? "0,0" : "POINT (0 0)").field("non_existent")).get(); + assertSearchResponse(response); + GeoBounds geoBounds = response.getAggregations().get("agg"); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo("agg")); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(0.0, GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(0.0, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); + } + + { + SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) + .addAggregation(geoBounds("agg").missing("LINESTRING (30 10, 10 30, 40 40)").field("non_existent")).get(); + assertSearchResponse(response); + GeoBounds geoBounds = response.getAggregations().get("agg"); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo("agg")); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(40, GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(10, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(10, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(40, GEOHASH_TOLERANCE)); + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java index 4f27cc1e3099e..57e56748d2f65 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java @@ -25,14 +25,29 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; +import org.elasticsearch.common.geo.CentroidCalculator; +import org.elasticsearch.common.geo.DimensionalShapeType; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoTestUtils; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomGeoGenerator; +import org.locationtech.spatial4j.exception.InvalidShapeException; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import static org.elasticsearch.common.geo.DimensionalShapeType.POINT; public class GeoCentroidAggregatorTests extends AggregatorTestCase { @@ -108,7 +123,7 @@ public void testUnmappedWithMissing() throws Exception { } } - public void testSingleValuedField() throws Exception { + public void testSingleValuedGeoPointField() throws Exception { int numDocs = scaledRandomIntBetween(64, 256); int numUniqueGeoPoints = randomIntBetween(1, numDocs); try (Directory dir = newDirectory(); @@ -127,11 +142,11 @@ public void testSingleValuedField() throws Exception { expectedCentroid = expectedCentroid.reset(expectedCentroid.lat() + (singleVal.lat() - expectedCentroid.lat()) / (i + 1), expectedCentroid.lon() + (singleVal.lon() - expectedCentroid.lon()) / (i + 1)); } - assertCentroid(w, expectedCentroid); + assertCentroid(w, expectedCentroid, new GeoPointFieldMapper.GeoPointFieldType()); } } - public void testMultiValuedField() throws Exception { + public void testMultiValuedGeoPointField() throws Exception { int numDocs = scaledRandomIntBetween(64, 256); int numUniqueGeoPoints = randomIntBetween(1, numDocs); try (Directory dir = newDirectory(); @@ -155,12 +170,61 @@ public void testMultiValuedField() throws Exception { expectedCentroid = expectedCentroid.reset(expectedCentroid.lat() + (newMVLat - expectedCentroid.lat()) / (i + 1), expectedCentroid.lon() + (newMVLon - expectedCentroid.lon()) / (i + 1)); } - assertCentroid(w, expectedCentroid); + assertCentroid(w, expectedCentroid, new GeoPointFieldMapper.GeoPointFieldType()); + } + } + + @SuppressWarnings("unchecked") + public void testGeoShapeField() throws Exception { + int numDocs = scaledRandomIntBetween(64, 256); + List geometries = new ArrayList<>(); + DimensionalShapeType targetShapeType = POINT; + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + for (int i = 0; i < numDocs; i++) { + Function geometryGenerator = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPoint, + (hasAlt) -> GeometryTestUtils.randomRectangle(), + GeometryTestUtils::randomMultiPolygon + ); + Geometry geometry = geometryGenerator.apply(false); + try { + geometries.add(indexer.prepareForIndexing(geometry)); + } catch (InvalidShapeException e) { + // do not include geometry + } + // find dimensional-shape-type of geometry + CentroidCalculator centroidCalculator = new CentroidCalculator(geometry); + DimensionalShapeType geometryShapeType = centroidCalculator.getDimensionalShapeType(); + targetShapeType = targetShapeType.compareTo(geometryShapeType) >= 0 ? targetShapeType : geometryShapeType; + } + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + CompensatedSum compensatedSumLon = new CompensatedSum(0, 0); + CompensatedSum compensatedSumLat = new CompensatedSum(0, 0); + CompensatedSum compensatedSumWeight = new CompensatedSum(0, 0); + for (Geometry geometry : geometries) { + Document document = new Document(); + CentroidCalculator calculator = new CentroidCalculator(geometry); + document.add(new BinaryGeoShapeDocValuesField("field", GeoTestUtils.toDecodedTriangles(geometry), calculator)); + w.addDocument(document); + if (targetShapeType.compareTo(calculator.getDimensionalShapeType()) == 0) { + double weight = calculator.sumWeight(); + compensatedSumLat.add(weight * calculator.getY()); + compensatedSumLon.add(weight * calculator.getX()); + compensatedSumWeight.add(weight); + } + } + GeoPoint expectedCentroid = new GeoPoint(compensatedSumLat.value() / compensatedSumWeight.value(), + compensatedSumLon.value() / compensatedSumWeight.value()); + assertCentroid(w, expectedCentroid, new GeoShapeFieldMapper.GeoShapeFieldType()); } } - private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) throws IOException { - MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType(); + private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid, MappedFieldType fieldType) throws IOException { fieldType.setHasDocValues(true); fieldType.setName("field"); GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg") diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java index 9e9af4e65066f..bb706872c134b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java @@ -19,8 +19,10 @@ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGrid; import org.elasticsearch.search.aggregations.bucket.global.Global; @@ -35,6 +37,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; @@ -45,10 +48,35 @@ public class GeoCentroidIT extends AbstractGeoTestCase { private static final String aggName = "geoCentroid"; + public void test7xIndexOnly() { + SearchPhaseExecutionException exception = expectThrows(SearchPhaseExecutionException.class, + () -> client().prepareSearch(IDX_NAME_7x) .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME)) + .get()); + assertNotNull(exception.getRootCause()); + assertThat(exception.getRootCause().getMessage(), + equalTo("Can't load fielddata on [geoshape_value] because fielddata is unsupported on fields of type [geo_shape]." + + " Use doc values instead.")); + } + + public void test7xIndexWith8Index() { + SearchResponse response = client().prepareSearch(IDX_NAME_7x, IDX_NAME) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME)) + .get(); + assertThat(response.status(), equalTo(RestStatus.OK)); + assertThat(response.getSuccessfulShards(), lessThan(response.getTotalShards())); + GeoCentroid geoCentroid = response.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); + assertEquals(numDocs, geoCentroid.count()); + } + public void testEmptyAggregation() throws Exception { SearchResponse response = client().prepareSearch(EMPTY_IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME)) .get(); assertSearchResponse(response); @@ -63,7 +91,7 @@ public void testEmptyAggregation() throws Exception { public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch(UNMAPPED_IDX_NAME) - .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME)) .get(); assertSearchResponse(response); @@ -77,7 +105,7 @@ public void testUnmapped() throws Exception { public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME, UNMAPPED_IDX_NAME) - .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME)) .get(); assertSearchResponse(response); @@ -93,7 +121,7 @@ public void testPartiallyUnmapped() throws Exception { public void testSingleValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME)) .get(); assertSearchResponse(response); @@ -106,10 +134,26 @@ public void testSingleValuedField() throws Exception { assertEquals(numDocs, geoCentroid.count()); } + public void testShapeField() throws Exception { + SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) + .setQuery(matchAllQuery()) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME)) + .get(); + assertSearchResponse(response); + + GeoCentroid geoCentroid = response.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid.lat(), closeTo(singleShapeCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(centroid.lon(), closeTo(singleShapeCentroid.lon(), GEOHASH_TOLERANCE)); + assertEquals(5, geoCentroid.count()); + } + public void testSingleValueFieldGetProperty() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(global("global").subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME))) + .addAggregation(global("global").subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME))) .get(); assertSearchResponse(response); @@ -123,7 +167,7 @@ public void testSingleValueFieldGetProperty() throws Exception { GeoCentroid geoCentroid = global.getAggregations().get(aggName); assertThat(geoCentroid, notNullValue()); assertThat(geoCentroid.getName(), equalTo(aggName)); - assertThat((GeoCentroid) ((InternalAggregation)global).getProperty(aggName), sameInstance(geoCentroid)); + assertThat(((InternalAggregation)global).getProperty(aggName), sameInstance(geoCentroid)); GeoPoint centroid = geoCentroid.centroid(); assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); @@ -154,8 +198,8 @@ public void testMultiValuedField() throws Exception { public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) - .addAggregation(geohashGrid("geoGrid").field(SINGLE_VALUED_FIELD_NAME) - .subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME))) + .addAggregation(geohashGrid("geoGrid").field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) + .subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME))) .get(); assertSearchResponse(response); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java index e3060d4c78ef0..cc84f7d8e99e7 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/ValueCountAggregatorTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -240,6 +241,14 @@ private static MappedFieldType createMappedFieldType(ValueType valueType) { return new GeoPointFieldMapper.Builder("_name").fieldType(); case RANGE: return new RangeFieldMapper.Builder("_name", RangeType.DOUBLE).fieldType(); + case GEOSHAPE: + return new GeoShapeFieldMapper.Builder("_name").fieldType(); + case GEO: + if (randomBoolean()) { + return new GeoPointFieldMapper.Builder("_name").fieldType(); + } else { + return new GeoShapeFieldMapper.Builder("_name").fieldType(); + } default: throw new IllegalArgumentException("Test does not support value type [" + valueType + "]"); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java index 5f57a8ee9c12e..126e1fd1d57cc 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceTypeTests.java @@ -39,6 +39,8 @@ public void testValidOrdinals() { assertThat(CoreValuesSourceType.GEOPOINT.ordinal(), equalTo(3)); assertThat(CoreValuesSourceType.RANGE.ordinal(), equalTo(4)); assertThat(CoreValuesSourceType.HISTOGRAM.ordinal(), equalTo(5)); + assertThat(CoreValuesSourceType.GEOSHAPE.ordinal(), equalTo(6)); + assertThat(CoreValuesSourceType.GEO.ordinal(), equalTo(7)); } @Override @@ -49,6 +51,8 @@ public void testFromString() { assertThat(CoreValuesSourceType.fromString("geopoint"), equalTo(CoreValuesSourceType.GEOPOINT)); assertThat(CoreValuesSourceType.fromString("range"), equalTo(CoreValuesSourceType.RANGE)); assertThat(CoreValuesSourceType.fromString("histogram"), equalTo(CoreValuesSourceType.HISTOGRAM)); + assertThat(CoreValuesSourceType.fromString("geoshape"), equalTo(CoreValuesSourceType.GEOSHAPE)); + assertThat(CoreValuesSourceType.fromString("geo"), equalTo(CoreValuesSourceType.GEO)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> CoreValuesSourceType.fromString("does_not_exist")); assertThat(e.getMessage(), @@ -64,6 +68,8 @@ public void testReadFrom() throws IOException { assertReadFromStream(3, CoreValuesSourceType.GEOPOINT); assertReadFromStream(4, CoreValuesSourceType.RANGE); assertReadFromStream(5, CoreValuesSourceType.HISTOGRAM); + assertReadFromStream(6, CoreValuesSourceType.GEOSHAPE); + assertReadFromStream(7, CoreValuesSourceType.GEO); } @Override @@ -74,5 +80,7 @@ public void testWriteTo() throws IOException { assertWriteToStream(CoreValuesSourceType.GEOPOINT, 3); assertWriteToStream(CoreValuesSourceType.RANGE, 4); assertWriteToStream(CoreValuesSourceType.HISTOGRAM, 5); + assertWriteToStream(CoreValuesSourceType.GEOSHAPE, 6); + assertWriteToStream(CoreValuesSourceType.GEO, 7); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index fb18cd9903235..0a18ae1349558 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -27,12 +27,18 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.TriangleTreeReader; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.geometry.Geometry; import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.geo.RandomShapeGenerator; import java.io.IOException; import java.util.Arrays; @@ -40,6 +46,8 @@ import java.util.Set; import java.util.function.LongUnaryOperator; +import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; + public class MissingValuesTests extends ESTestCase { public void testMissingBytes() throws IOException { @@ -346,20 +354,76 @@ public int docValueCount() { public void testMissingGeoPoints() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); - final GeoPoint[][] values = new GeoPoint[numDocs][]; + final MultiGeoValues.GeoPointValue[][] values = new MultiGeoValues.GeoPointValue[numDocs][]; + for (int i = 0; i < numDocs; ++i) { + values[i] = new MultiGeoValues.GeoPointValue[random().nextInt(4)]; + for (int j = 0; j < values[i].length; ++j) { + values[i][j] = new MultiGeoValues.GeoPointValue(new GeoPoint(randomDouble() * 90, randomDouble() * 180)); + } + } + MultiGeoValues asGeoValues = new MultiGeoValues() { + + int doc = -1; + int i; + + @Override + public GeoValue nextValue() { + return values[doc][i++]; + } + + @Override + public boolean advanceExact(int docId) { + doc = docId; + i = 0; + return values[doc].length > 0; + } + + @Override + public int docValueCount() { + return values[doc].length; + } + + @Override + public ValuesSourceType valuesSourceType() { + return CoreValuesSourceType.GEOPOINT; + } + }; + final MultiGeoValues.GeoPointValue missing = new MultiGeoValues.GeoPointValue( + new GeoPoint(randomDouble() * 90, randomDouble() * 180)); + MultiGeoValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); + for (int i = 0; i < numDocs; ++i) { + assertTrue(withMissingReplaced.advanceExact(i)); + if (values[i].length > 0) { + assertEquals(values[i].length, withMissingReplaced.docValueCount()); + for (int j = 0; j < values[i].length; ++j) { + assertEquals(values[i][j], withMissingReplaced.nextValue()); + } + } else { + assertEquals(1, withMissingReplaced.docValueCount()); + assertEquals(missing, withMissingReplaced.nextValue()); + } + } + } + + public void testMissingGeoShapes() throws IOException { + final int numDocs = TestUtil.nextInt(random(), 1, 100); + final MultiGeoValues.GeoShapeValue[][] values = new MultiGeoValues.GeoShapeValue[numDocs][]; for (int i = 0; i < numDocs; ++i) { - values[i] = new GeoPoint[random().nextInt(4)]; + values[i] = new MultiGeoValues.GeoShapeValue[random().nextInt(4)]; for (int j = 0; j < values[i].length; ++j) { - values[i][j] = new GeoPoint(randomDouble() * 90, randomDouble() * 180); + ShapeBuilder builder = RandomShapeGenerator.createShape(random()); + Geometry geometry = new GeoShapeIndexer(true, "test").prepareForIndexing(builder.buildGeometry()); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + values[i][j] = new MultiGeoValues.GeoShapeValue(reader); } } - MultiGeoPointValues asGeoValues = new MultiGeoPointValues() { + MultiGeoValues asGeoValues = new MultiGeoValues() { int doc = -1; int i; @Override - public GeoPoint nextValue() { + public GeoValue nextValue() { return values[doc][i++]; } @@ -374,9 +438,16 @@ public boolean advanceExact(int docId) { public int docValueCount() { return values[doc].length; } + + @Override + public ValuesSourceType valuesSourceType() { + return CoreValuesSourceType.GEOSHAPE; + } }; - final GeoPoint missing = new GeoPoint(randomDouble() * 90, randomDouble() * 180); - MultiGeoPointValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); + ShapeBuilder builder = RandomShapeGenerator.createShape(random()); + TriangleTreeReader reader = triangleTreeReader(builder.buildGeometry(), GeoShapeCoordinateEncoder.INSTANCE); + final MultiGeoValues.GeoShapeValue missing = new MultiGeoValues.GeoShapeValue(reader); + MultiGeoValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); for (int i = 0; i < numDocs; ++i) { assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java index db208044df1a4..163126bfeeaba 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java @@ -23,14 +23,21 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.TypeFieldMapper; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.test.ESSingleNodeTestCase; +import java.io.IOException; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; + public class ValuesSourceConfigTests extends ESSingleNodeTestCase { public void testKeyword() throws Exception { @@ -260,6 +267,180 @@ public void testUnmappedBoolean() throws Exception { } } + public void testGeoPoint() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type", + "geo_point", "type=geo_point"); + client().prepareIndex("index") + .setSource("geo_point", "-10.0,10.0") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "geo_point", null, null, null, null); + ValuesSource.GeoPoint valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + MultiGeoValues.GeoValue value = values.nextValue(); + assertThat(value.lat(), closeTo(-10, GeoUtils.TOLERANCE)); + assertThat(value.lon(), closeTo(10, GeoUtils.TOLERANCE)); + } + } + + public void testEmptyGeoPoint() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type", + "geo_point", "type=geo_point"); + client().prepareIndex("index") + .setSource() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "geo_point", null, null, null, null); + ValuesSource.GeoPoint valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertFalse(values.advanceExact(0)); + + config = ValuesSourceConfig.resolve( + context, null, "geo_point", null, "0,0", null, null); + valuesSource = config.toValuesSource(context); + ctx = searcher.getIndexReader().leaves().get(0); + values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + MultiGeoValues.GeoValue value = values.nextValue(); + assertThat(value.lat(), closeTo(0, GeoUtils.TOLERANCE)); + assertThat(value.lon(), closeTo(0, GeoUtils.TOLERANCE)); + } + } + + public void testUnmappedGeoPoint() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type"); + client().prepareIndex("index") + .setSource() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, ValueType.GEOPOINT, "geo_point", null, null, null, null); + ValuesSource.GeoPoint valuesSource = config.toValuesSource(context); + assertNull(valuesSource); + + config = ValuesSourceConfig.resolve( + context, ValueType.GEOPOINT, "geo_point", null, "0,0", null, null); + valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + MultiGeoValues.GeoValue value = values.nextValue(); + assertThat(value.lat(), closeTo(0, GeoUtils.TOLERANCE)); + assertThat(value.lon(), closeTo(0, GeoUtils.TOLERANCE)); + } + } + + public void testGeoShape() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type", + "geo_shape", "type=geo_shape"); + client().prepareIndex("index") + .setSource("geo_shape", "POINT (-10 10)") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "geo_shape", null, null, null, null); + ValuesSource.GeoShape valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + // TODO (talevy): assert value once BoundingBox is defined +// MultiGeoValues.GeoValue value = values.nextValue(); +// assertThat(value.minX(), closeTo(-10, GeoUtils.TOLERANCE)); +// assertThat(value.minY(), closeTo(10, GeoUtils.TOLERANCE)); + } + } + + public void testEmptyGeoShape() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type", + "geo_shape", "type=geo_shape"); + client().prepareIndex("index") + .setSource() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "geo_shape", null, null, null, null); + ValuesSource.GeoShape valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertFalse(values.advanceExact(0)); + + config = ValuesSourceConfig.resolve( + context, null, "geo_shape", null, "POINT (0 0)", null, null); + valuesSource = config.toValuesSource(context); + ctx = searcher.getIndexReader().leaves().get(0); + values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + // TODO (talevy): assert value once BoundingBox is defined +// MultiGeoValues.GeoValue value = values.nextValue(); +// assertThat(value.minX(), closeTo(-10, GeoUtils.TOLERANCE)); +// assertThat(value.minY(), closeTo(10, GeoUtils.TOLERANCE)); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> ValuesSourceConfig.resolve(context, ValueType.GEO, "geo_shapes", null, "invalid", + null, null).toValuesSource(context)); + assertThat(exception.getMessage(), equalTo("Unknown geometry type: invalid")); + } + } + + public void testUnmappedGeoShape() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type"); + client().prepareIndex("index") + .setSource() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher, () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, ValueType.GEOSHAPE, "geo_shape", null, null, null, null); + ValuesSource.GeoShape valuesSource = config.toValuesSource(context); + assertNull(valuesSource); + + config = ValuesSourceConfig.resolve( + context, ValueType.GEOSHAPE, "geo_shape", null, "POINT (0 0)", null, null); + valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + // TODO (talevy): assert value once BoundingBox is defined +// MultiGeoValues.GeoValue value = values.nextValue(); +// assertThat(value.minX(), closeTo(-10, GeoUtils.TOLERANCE)); +// assertThat(value.minY(), closeTo(10, GeoUtils.TOLERANCE)); + } + } + public void testTypeFieldDeprecation() { IndexService indexService = createIndex("index", Settings.EMPTY, "type"); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { diff --git a/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java b/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java index e4522a37e486a..5cf18929ec672 100644 --- a/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java @@ -20,6 +20,7 @@ package org.elasticsearch.geo; import org.apache.lucene.geo.GeoTestUtil; +import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; @@ -258,4 +259,66 @@ public MultiPoint visit(Rectangle rectangle) throws RuntimeException { } }); } + + /** + * Preforms left fold operation on all primitive geometries (points, lines polygons, circles and rectangles). + * All collection geometries are iterated depth first. + */ + public static R fold(Geometry geometry, R state, CheckedBiFunction operation) throws E { + return geometry.visit(new GeometryVisitor() { + @Override + public R visit(Circle circle) throws E { + return operation.apply(geometry, state); + } + + @Override + public R visit(GeometryCollection collection) throws E { + R ret = state; + for (Geometry g : collection) { + ret = fold(g, ret, operation); + } + return ret; + } + + @Override + public R visit(Line line) throws E { + return operation.apply(line, state); + } + + @Override + public R visit(LinearRing ring) throws E { + return operation.apply(ring, state); + } + + @Override + public R visit(MultiLine multiLine) throws E { + return visit((GeometryCollection) multiLine); + } + + @Override + public R visit(MultiPoint multiPoint) throws E { + return visit((GeometryCollection) multiPoint); } + + @Override + public R visit(MultiPolygon multiPolygon) throws E { + return visit((GeometryCollection) multiPolygon); + } + + @Override + public R visit(Point point) throws E { + return operation.apply(point, state); + } + + @Override + public R visit(Polygon polygon) throws E { + return operation.apply(polygon, state); + } + + @Override + public R visit(Rectangle rectangle) throws E { + return operation.apply(rectangle, state); + } + }); + } + } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java index 5d003dd80bf08..87850106b9081 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/index/IndexResolver.java @@ -590,4 +590,4 @@ private static List buildIndices(DataTypeRegistry typeRegistry, String[ foundIndices.sort(Comparator.comparing(EsIndex::name)); return foundIndices; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java index 6253a5f89d7e2..8dc322d6bcc0a 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/type/DataTypes.java @@ -46,7 +46,7 @@ public final class DataTypes { public static final DataType OBJECT = new DataType("object", 0, false, false, false); public static final DataType NESTED = new DataType("nested", 0, false, false, false); //@formatter:on - + private static final Collection TYPES = Arrays.asList( UNSUPPORTED, NULL, diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 472caa6e8ed86..7768e3ed0804a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.spatial.index.mapper; import org.apache.lucene.document.XYShape; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.geo.builders.ShapeBuilder; @@ -53,7 +54,16 @@ public Builder(String name) { public ShapeFieldMapper build(BuilderContext context) { setupFieldType(context); return new ShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), - ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + ignoreZValue(), docValues(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + } + + public ShapeFieldMapper.Builder docValues(boolean hasDocValues) { + super.docValues(hasDocValues); + if (hasDocValues) { + throw new ElasticsearchParseException("field [" + name + "] of type [" + fieldType().typeName() + + "] does not support doc-values"); + } + return this; } @Override @@ -116,9 +126,9 @@ protected Indexer geometryIndexer() { public ShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, - Explicit ignoreZValue, Settings indexSettings, + Explicit ignoreZValue, Explicit docValues, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, indexSettings, multiFields, copyTo); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java index 2df89bee955ba..5e0e47cb7a5d5 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java @@ -5,10 +5,12 @@ */ package org.elasticsearch.xpack.spatial.index.mapper; +import org.apache.lucene.document.ShapeField; import org.apache.lucene.document.XYShape; import org.apache.lucene.geo.XYLine; import org.apache.lucene.geo.XYPolygon; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; @@ -22,6 +24,7 @@ import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.ParseContext; import java.util.ArrayList; @@ -52,6 +55,18 @@ public List indexShape(ParseContext context, Geometry shape) { return visitor.fields; } + @Override + public void indexDocValueField(ParseContext context, ShapeField.DecodedTriangle[] triangles, + CentroidCalculator centroidCalculator) { + BinaryGeoShapeDocValuesField docValuesField = (BinaryGeoShapeDocValuesField) context.doc().getByKey(name); + if (docValuesField == null) { + docValuesField = new BinaryGeoShapeDocValuesField(name, triangles, centroidCalculator); + context.doc().addWithKey(name, docValuesField); + } else { + docValuesField.add(triangles, centroidCalculator); + } + } + private class LuceneGeometryVisitor implements GeometryVisitor { private List fields = new ArrayList<>(); private String name; diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 809f9d621395e..d8012a6ea2f42 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.spatial.index.mapper; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -54,6 +55,9 @@ public void testDefaultConfiguration() throws IOException { ShapeFieldMapper shapeFieldMapper = (ShapeFieldMapper) fieldMapper; assertThat(shapeFieldMapper.fieldType().orientation(), equalTo(ShapeFieldMapper.Defaults.ORIENTATION.value())); + assertFalse(shapeFieldMapper.docValues().value()); + assertFalse(shapeFieldMapper.docValues().explicit()); + assertFalse(shapeFieldMapper.fieldType().hasDocValues()); } /** @@ -96,6 +100,40 @@ public void testOrientationParsing() throws IOException { assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); } + /** + * Test that doc_values parameter correctly parses + */ + public void testDocValues() throws IOException { + String trueMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "shape") + .field("doc_values", true) + .endObject().endObject() + .endObject().endObject()); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(trueMapping))); + assertThat(e.getMessage(), equalTo("field [location] of type [shape] does not support doc-values")); + + // explicit false doc_values + String falseMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "shape") + .field("doc_values", false) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(falseMapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); + + // since shape field has no doc-values, this field is ignored + assertTrue(((ShapeFieldMapper)fieldMapper).docValues().explicit()); + assertFalse(((ShapeFieldMapper)fieldMapper).docValues().value()); + assertFalse(((ShapeFieldMapper)fieldMapper).fieldType().hasDocValues()); + } + /** * Test that coerce parameter correctly parses */ @@ -276,9 +314,25 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((ShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"orientation\":\"" + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } } + public void testSerializeDocValues() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "shape") + .field("doc_values", false) + .endObject().endObject() + .endObject().endObject()); + DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((ShapeFieldMapper) mapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); + } + public String toXContentString(ShapeFieldMapper mapper, boolean includeDefaults) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); ToXContent.Params params; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java index 6c321bea80de8..4bbf70a810e50 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/SqlDataTypes.java @@ -80,7 +80,7 @@ public class SqlDataTypes { public static final DataType INTERVAL_MINUTE_TO_SECOND = new DataType("INTERVAL_MINUTE_TO_SECOND", null, Long.BYTES, false, false, false); // geo - public static final DataType GEO_SHAPE = new DataType("geo_shape", Integer.MAX_VALUE, false, false, false); + public static final DataType GEO_SHAPE = new DataType("geo_shape", Integer.MAX_VALUE, false, false, true); public static final DataType GEO_POINT = new DataType("geo_point", Double.BYTES * 2, false, false, false); public static final DataType SHAPE = new DataType("shape", Integer.MAX_VALUE, false, false, false); // @formatter:on @@ -269,7 +269,6 @@ public static boolean isFromDocValuesOnly(DataType dataType) { || dataType == DATETIME || dataType == SCALED_FLOAT // because of scaling_factor || dataType == GEO_POINT - || dataType == GEO_SHAPE || dataType == SHAPE; }