From 00f5a6044f2ce3fc57c454c1bb307e03be2a95a2 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 21 May 2019 11:08:29 -0700 Subject: [PATCH 01/62] introduce EdgeTree writer and reader (#40810) This commit introduces a new data-structure for reading and writing EdgeTrees that write/read serialized versions of the tree. This tree is the basis of Polygon trees that will contain representation of any holes in the more complex polygon --- .../common/geo/EdgeTreeReader.java | 193 ++++++++++++++++++ .../common/geo/EdgeTreeWriter.java | 162 +++++++++++++++ .../io/stream/ByteBufferStreamInput.java | 8 + .../common/geo/EdgeTreeTests.java | 135 ++++++++++++ 4 files changed, 498 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java create mode 100644 server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java new file mode 100644 index 0000000000000..250e98e09d114 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -0,0 +1,193 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.apache.lucene.geo.GeoUtils.lineCrossesLine; + +public class EdgeTreeReader { + final BytesRef bytesRef; + + public EdgeTreeReader(BytesRef bytesRef) { + this.bytesRef = bytesRef; + } + + /** + * Returns true if the rectangle query and the edge tree's shape overlap + */ + public boolean containedInOrCrosses(int minX, int minY, int maxX, int maxY) throws IOException { + return this.containsBottomLeft(minX, minY, maxX, maxY) || this.crosses(minX, minY, maxX, maxY); + } + + boolean containsBottomLeft(int minX, int minY, int maxX, int maxY) throws IOException { + ByteBufferStreamInput input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + int thisMinX = input.readInt(); + int thisMinY = input.readInt(); + int thisMaxX = input.readInt(); + int thisMaxY = input.readInt(); + + if (thisMinY > maxY || thisMaxX < minX || thisMaxY < minY || thisMinX > maxX) { + return false; // tree and bbox-query are disjoint + } + + if (minX <= thisMinX && minY <= thisMinY && maxX >= thisMaxX && maxY >= thisMaxY) { + return true; // bbox-query fully contains tree's extent. + } + + return containsBottomLeft(input, readRoot(input, input.position()), minX, minY, maxX, maxY); + } + + public boolean crosses(int minX, int minY, int maxX, int maxY) throws IOException { + ByteBufferStreamInput input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + int thisMinX = input.readInt(); + int thisMinY = input.readInt(); + int thisMaxX = input.readInt(); + int thisMaxY = input.readInt(); + + if (thisMinY > maxY || thisMaxX < minX || thisMaxY < minY || thisMinX > maxX) { + return false; // tree and bbox-query are disjoint + } + + if (minX <= thisMinX && minY <= thisMinY && maxX >= thisMaxX && maxY >= thisMaxY) { + return true; // bbox-query fully contains tree's extent. + } + + return crosses(input, readRoot(input, input.position()), minX, minY, maxX, maxY); + } + + public Edge readRoot(ByteBufferStreamInput input, int position) throws IOException { + return readEdge(input, position); + } + + private static Edge readEdge(ByteBufferStreamInput input, int position) throws IOException { + input.position(position); + int minY = input.readInt(); + int maxY = input.readInt(); + int x1 = input.readInt(); + int y1 = input.readInt(); + int x2 = input.readInt(); + int y2 = input.readInt(); + int rightOffset = input.readInt(); + return new Edge(input.position(), x1, y1, x2, y2, minY, maxY, rightOffset); + } + + + Edge readLeft(ByteBufferStreamInput input, Edge root) throws IOException { + return readEdge(input, root.streamOffset); + } + + Edge readRight(ByteBufferStreamInput input, Edge root) throws IOException { + return readEdge(input, root.streamOffset + root.rightOffset); + } + + /** + * Returns true if the bottom-left point of the rectangle query is contained within the + * tree's edges. + */ + private boolean containsBottomLeft(ByteBufferStreamInput input, Edge root, int minX, int minY, int maxX, int maxY) throws IOException { + boolean res = false; + if (root.maxY >= minY) { + // is bbox-query contained within linearRing + // cast infinite ray to the right from bottom-left of bbox-query to see if it intersects edge + if (lineCrossesLine(root.x1, root.y1, root.x2, root.y2,minX, minY, Integer.MAX_VALUE, minY)) { + res = true; + } + + if (root.rightOffset > 0) { /* has left node */ + res ^= containsBottomLeft(input, readLeft(input, root), minX, minY, maxX, maxY); + } + + if (root.rightOffset > 0 && maxY >= root.minY) { /* no right node if rightOffset == -1 */ + res ^= containsBottomLeft(input, readRight(input, root), minX, minY, maxX, maxY); + } + } + return res; + } + + /** + * Returns true if the box crosses any edge in this edge subtree + * */ + private boolean crosses(ByteBufferStreamInput input, Edge root, int minX, int minY, int maxX, int maxY) throws IOException { + boolean res = false; + // we just have to cross one edge to answer the question, so we descend the tree and return when we do. + if (root.maxY >= minY) { + + // does rectangle's edges intersect or reside inside polygon's edge + if (lineCrossesLine(root.x1, root.y1, root.x2, root.y2, minX, minY, maxX, minY) || + lineCrossesLine(root.x1, root.y1, root.x2, root.y2, maxX, minY, maxX, maxY) || + lineCrossesLine(root.x1, root.y1, root.x2, root.y2, maxX, maxY, minX, maxY) || + lineCrossesLine(root.x1, root.y1, root.x2, root.y2, minX, maxY, minX, minY)) { + return true; + } + + if (root.rightOffset > 0) { /* has left node */ + if (crosses(input, readLeft(input, root), minX, minY, maxX, maxY)) { + return true; + } + } + + if (root.rightOffset > 0 && maxY >= root.minY) { /* no right node if rightOffset == -1 */ + if (crosses(input, readRight(input, root), minX, minY, maxX, maxY)) { + return true; + } + } + } + return false; + } + + + private static class Edge { + int streamOffset; + int x1; + int y1; + int x2; + int y2; + int minY; + int maxY; + int rightOffset; + + /** + * Object representing an edge node read from bytes + * + * @param streamOffset offset in byte-reference where edge terminates + * @param x1 x-coordinate of first point in segment + * @param y1 y-coordinate of first point in segment + * @param x2 x-coordinate of second point in segment + * @param y2 y-coordinate of second point in segment + * @param minY minimum y-coordinate in this edge-node's tree + * @param maxY maximum y-coordinate in this edge-node's tree + * @param rightOffset the start offset in the byte-reference of the right edge-node + */ + Edge(int streamOffset, int x1, int y1, int x2, int y2, int minY, int maxY, int rightOffset) { + this.streamOffset = streamOffset; + this.x1 = x1; + this.y1 = y1; + this.x2 = x2; + this.y2 = y2; + this.minY = minY; + this.maxY = maxY; + this.rightOffset = rightOffset; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java new file mode 100644 index 0000000000000..4ffba80a12817 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -0,0 +1,162 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; +import java.util.Arrays; + +/** + * Shape edge-tree writer for use in doc-values + */ +public class EdgeTreeWriter { + + /** + * | minY | maxY | x1 | y1 | x2 | y2 | right_offset | + */ + static final int EDGE_SIZE_IN_BYTES = 28; + + int minX; + int minY; + int maxX; + int maxY; + final Edge tree; + + public EdgeTreeWriter(int[] x, int[] y) { + minX = minY = Integer.MAX_VALUE; + maxX = maxY = Integer.MIN_VALUE; + Edge edges[] = new Edge[y.length - 1]; + for (int i = 1; i < y.length; i++) { + int y1 = y[i-1]; + int x1 = x[i-1]; + int y2 = y[i]; + int x2 = x[i]; + int minY, maxY; + if (y1 < y2) { + minY = y1; + maxY = y2; + } else { + minY = y2; + maxY = y1; + } + edges[i - 1] = new Edge(x1, y1, x2, y2, minY, maxY); + this.minX = Math.min(this.minX, Math.min(x1, x2)); + this.minY = Math.min(this.minY, Math.min(y1, y2)); + this.maxX = Math.max(this.maxX, Math.max(x1, x2)); + this.maxY = Math.max(this.maxY, Math.max(y1, y2)); + } + Arrays.sort(edges); + this.tree = createTree(edges, 0, edges.length - 1); + } + + public BytesRef toBytesRef() throws IOException { + BytesStreamOutput output = new BytesStreamOutput(4 * 4 + EDGE_SIZE_IN_BYTES * tree.size); + // write extent of edges + output.writeInt(minX); + output.writeInt(minY); + output.writeInt(maxX); + output.writeInt(maxY); + // write edge-tree itself + writeTree(tree, output); + output.close(); + return output.bytes().toBytesRef(); + } + + private void writeTree(Edge edge, StreamOutput output) throws IOException { + if (edge == null) { + return; + } + output.writeInt(edge.minY); + output.writeInt(edge.maxY); + output.writeInt(edge.x1); + output.writeInt(edge.y1); + output.writeInt(edge.x2); + output.writeInt(edge.y2); + // left node is next node, write offset of right node + if (edge.left != null) { + output.writeInt(edge.left.size * EDGE_SIZE_IN_BYTES); + } else if (edge.right == null){ + output.writeInt(-1); + } else { + output.writeInt(0); + } + writeTree(edge.left, output); + writeTree(edge.right, output); + } + + private static Edge createTree(Edge edges[], int low, int high) { + if (low > high) { + return null; + } + // add midpoint + int mid = (low + high) >>> 1; + Edge newNode = edges[mid]; + newNode.size = 1; + // add children + newNode.left = createTree(edges, low, mid - 1); + newNode.right = createTree(edges, mid + 1, high); + // pull up max values to this node + // and node count + if (newNode.left != null) { + newNode.maxY = Math.max(newNode.maxY, newNode.left.maxY); + newNode.size += newNode.left.size; + } + if (newNode.right != null) { + newNode.maxY = Math.max(newNode.maxY, newNode.right.maxY); + newNode.size += newNode.right.size; + } + return newNode; + } + + /** + * Object representing an in-memory edge-tree to be serialized + */ + static class Edge implements Comparable { + final int x1; + final int y1; + final int x2; + final int y2; + int minY; + int maxY; + int size; + Edge left; + Edge right; + + Edge(int x1, int y1, int x2, int y2, int minY, int maxY) { + this.x1 = x1; + this.y1 = y1; + this.x2 = x2; + this.y2 = y2; + this.minY = minY; + this.maxY = maxY; + } + + @Override + public int compareTo(Edge other) { + int ret = Integer.compare(minY, other.minY); + if (ret == 0) { + ret = Integer.compare(maxY, other.maxY); + } + return ret; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java b/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java index 0668fcb85fef9..d245f3950433e 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/ByteBufferStreamInput.java @@ -110,6 +110,14 @@ public long readLong() throws IOException { } } + public void position(int newPosition) throws IOException { + buffer.position(newPosition); + } + + public int position() throws IOException { + return buffer.position(); + } + @Override public void reset() throws IOException { buffer.reset(); diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java new file mode 100644 index 0000000000000..386c439c15527 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -0,0 +1,135 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.geo.geometry.Polygon; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.geo.RandomShapeGenerator; +import org.locationtech.spatial4j.shape.Rectangle; + +import java.io.IOException; + +public class EdgeTreeTests extends ESTestCase { + + public void testRectangleShape() throws IOException { + for (int i = 0; i < 1000; i++) { + int minX = randomIntBetween(-180, 170); + int maxX = randomIntBetween(minX + 10, 180); + int minY = randomIntBetween(-180, 170); + int maxY = randomIntBetween(minY + 10, 180); + int[] x = new int[]{minX, maxX, maxX, minX, minX}; + int[] y = new int[]{minY, minY, maxY, maxY, minY}; + EdgeTreeWriter writer = new EdgeTreeWriter(x, y); + BytesRef bytes = writer.toBytesRef(); + EdgeTreeReader reader = new EdgeTreeReader(bytes); + + // box-query touches bottom-left corner + assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY)); + // box-query touches bottom-right corner + assertTrue(reader.containedInOrCrosses(maxX, minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), minY)); + // box-query touches top-right corner + assertTrue(reader.containedInOrCrosses(maxX, maxY, maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + // box-query touches top-left corner + assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), maxY, minX, maxY + randomIntBetween(1, 180))); + // box-query fully-enclosed inside rectangle + assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + (3 * maxY + minY) / 4)); + // box-query fully-contains poly + assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), + maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + // box-query half-in-half-out-right + assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + (3 * maxY + minY) / 4)); + // box-query half-in-half-out-left + assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + (3 * maxY + minY) / 4)); + // box-query half-in-half-out-top + assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + maxY + randomIntBetween(1, 1000))); + // box-query half-in-half-out-bottom + assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + + // box-query outside to the right + assertFalse(reader.containedInOrCrosses(maxX + randomIntBetween(1, 1000), minY, maxX + randomIntBetween(1001, 2000), maxY)); + // box-query outside to the left + assertFalse(reader.containedInOrCrosses(maxX - randomIntBetween(1001, 2000), minY, minX - randomIntBetween(1, 1000), maxY)); + // box-query outside to the top + assertFalse(reader.containedInOrCrosses(minX, maxY + randomIntBetween(1, 1000), maxX, maxY + randomIntBetween(1001, 2000))); + // box-query outside to the bottom + assertFalse(reader.containedInOrCrosses(minX, minY - randomIntBetween(1001, 2000), maxX, minY - randomIntBetween(1, 1000))); + } + } + + public void testSimplePolygon() throws IOException { + for (int iter = 0; iter < 1000; iter++) { + ShapeBuilder builder = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON); + Polygon geo = (Polygon) builder.buildGeometry(); + Rectangle box = builder.buildS4J().getBoundingBox(); + int minXBox = (int) box.getMinX(); + int minYBox = (int) box.getMinY(); + int maxXBox = (int) box.getMaxX(); + int maxYBox = (int) box.getMaxY(); + + int[] x = asIntArray(geo.getPolygon().getLons()); + int[] y = asIntArray(geo.getPolygon().getLats()); + + EdgeTreeWriter writer = new EdgeTreeWriter(x, y); + EdgeTreeReader reader = new EdgeTreeReader(writer.toBytesRef()); + // polygon fully contained within box + assertTrue(reader.containedInOrCrosses(minXBox, minYBox, maxXBox, maxYBox)); + // containedInOrCrosses + if (maxYBox - 1 >= minYBox) { + assertTrue(reader.containedInOrCrosses(minXBox, minYBox, maxXBox, maxYBox - 1)); + } + if (maxXBox -1 >= minXBox) { + assertTrue(reader.containedInOrCrosses(minXBox, minYBox, maxXBox - 1, maxYBox)); + } + // does not cross + assertFalse(reader.containedInOrCrosses(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10)); + } + } + + public void testPacMan() throws Exception { + // pacman + int[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + int[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // candidate containedInOrCrosses cell + int xMin = 2;//-5; + int xMax = 11;//0.000001; + int yMin = -1;//0; + int yMax = 1;//5; + + // test cell crossing poly + EdgeTreeWriter writer = new EdgeTreeWriter(px, py); + EdgeTreeReader reader = new EdgeTreeReader(writer.toBytesRef()); + assertTrue(reader.containsBottomLeft(xMin, yMin, xMax, yMax)); + } + + private int[] asIntArray(double[] doub) { + int[] intArr = new int[doub.length]; + for (int i = 0; i < intArr.length; i++) { + intArr[i] = (int) doub[i]; + } + return intArr; + } +} From e81ef33d302290d28e4fca8cbb9b925bd9136e25 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 22 May 2019 14:56:43 -0700 Subject: [PATCH 02/62] Introduce GeometryTree writer/reader (#42331) The GeometryTree represent an Elastisearch Geometry object. This includes collections like MultiPoint and GeometryCollection. For the initial implementation, only polygons without holes are supported. In a follow-up PR, the GeometryTree will be the object that interacts with doc-value reading and writing. --- .../common/geo/GeometryTreeReader.java | 74 ++++++++ .../common/geo/GeometryTreeWriter.java | 174 ++++++++++++++++++ .../common/geo/GeometryTreeTests.java | 97 ++++++++++ 3 files changed, 345 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java create mode 100644 server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java new file mode 100644 index 0000000000000..f709a3f20a598 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -0,0 +1,74 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.geo.geometry.ShapeType; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * A tree reader. + * + * This class supports checking bounding box + * relations against the serialized geometry tree. + */ +public class GeometryTreeReader { + + private final BytesRef bytesRef; + + public GeometryTreeReader(BytesRef bytesRef) { + this.bytesRef = bytesRef; + } + + public boolean containedInOrCrosses(int minLon, int minLat, int maxLon, int maxLat) throws IOException { + ByteBufferStreamInput input = new ByteBufferStreamInput( + ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + boolean hasExtent = input.readBoolean(); + if (hasExtent) { + int thisMinLon = input.readInt(); + int thisMinLat = input.readInt(); + int thisMaxLon = input.readInt(); + int thisMaxLat = input.readInt(); + + if (thisMinLat > maxLat || thisMaxLon < minLon || thisMaxLat < minLat || thisMinLon > maxLon) { + return false; // tree and bbox-query are disjoint + } + + if (minLon <= thisMinLon && minLat <= thisMinLat && maxLon >= thisMaxLon && maxLat >= thisMaxLat) { + return true; // bbox-query fully contains tree's extent. + } + } + + int numTrees = input.readVInt(); + for (int i = 0; i < numTrees; i++) { + ShapeType shapeType = input.readEnum(ShapeType.class); + if (ShapeType.POLYGON.equals(shapeType)) { + BytesRef treeRef = input.readBytesRef(); + EdgeTreeReader reader = new EdgeTreeReader(treeRef); + if (reader.containedInOrCrosses(minLon, minLat, maxLon, maxLat)) { + return true; + } + } + } + return false; + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java new file mode 100644 index 0000000000000..21de9b7f7601f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -0,0 +1,174 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.geometry.Circle; +import org.elasticsearch.geo.geometry.Geometry; +import org.elasticsearch.geo.geometry.GeometryCollection; +import org.elasticsearch.geo.geometry.GeometryVisitor; +import org.elasticsearch.geo.geometry.Line; +import org.elasticsearch.geo.geometry.LinearRing; +import org.elasticsearch.geo.geometry.MultiLine; +import org.elasticsearch.geo.geometry.MultiPoint; +import org.elasticsearch.geo.geometry.MultiPolygon; +import org.elasticsearch.geo.geometry.Point; +import org.elasticsearch.geo.geometry.Polygon; +import org.elasticsearch.geo.geometry.Rectangle; +import org.elasticsearch.geo.geometry.ShapeType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * This is a tree-writer that serializes the + * appropriate tree structure for each type of + * {@link Geometry} into a byte array. + */ +public class GeometryTreeWriter { + + private final GeometryTreeBuilder builder; + + GeometryTreeWriter(Geometry geometry) { + builder = new GeometryTreeBuilder(); + geometry.visit(builder); + } + + public BytesRef toBytesRef() throws IOException { + BytesStreamOutput output = new BytesStreamOutput(); + // only write a geometry extent for the tree if the tree + // contains multiple sub-shapes + boolean prependExtent = builder.shapeWriters.size() > 1; + output.writeBoolean(prependExtent); + if (prependExtent) { + output.writeInt(builder.minLon); + output.writeInt(builder.minLat); + output.writeInt(builder.maxLon); + output.writeInt(builder.maxLat); + } + output.writeVInt(builder.shapeWriters.size()); + for (EdgeTreeWriter writer : builder.shapeWriters) { + output.writeEnum(ShapeType.POLYGON); + output.writeBytesRef(writer.toBytesRef()); + } + output.close(); + return output.bytes().toBytesRef(); + } + + class GeometryTreeBuilder implements GeometryVisitor { + + private List shapeWriters; + // integers are used to represent int-encoded lat/lon values + int minLat; + int maxLat; + int minLon; + int maxLon; + + GeometryTreeBuilder() { + shapeWriters = new ArrayList<>(); + minLat = minLon = Integer.MAX_VALUE; + maxLat = maxLon = Integer.MIN_VALUE; + } + + private void addWriter(EdgeTreeWriter writer) { + minLon = Math.min(minLon, writer.minX); + minLat = Math.min(minLat, writer.minY); + maxLon = Math.max(maxLon, writer.maxX); + maxLat = Math.max(maxLat, writer.maxY); + shapeWriters.add(writer); + } + + @Override + public Void visit(GeometryCollection collection) { + for (Geometry geometry : collection) { + geometry.visit(this); + } + return null; + } + + @Override + public Void visit(Line line) { + throw new UnsupportedOperationException("support for Line is a TODO"); + } + + @Override + public Void visit(MultiLine multiLine) { + for (Line line : multiLine) { + visit(line); + } + return null; + } + + @Override + public Void visit(Polygon polygon) { + // TODO (support holes) + LinearRing outerShell = polygon.getPolygon(); + addWriter(new EdgeTreeWriter(asIntArray(outerShell.getLons()), asIntArray(outerShell.getLats()))); + return null; + } + + @Override + public Void visit(MultiPolygon multiPolygon) { + for (Polygon polygon : multiPolygon) { + visit(polygon); + } + return null; + } + + @Override + public Void visit(Rectangle r) { + int[] lats = new int[] { (int) r.getMinLat(), (int) r.getMinLat(), (int) r.getMaxLat(), (int) r.getMaxLat(), + (int) r.getMinLat()}; + int[] lons = new int[] { (int) r.getMinLon(), (int) r.getMaxLon(), (int) r.getMaxLon(), (int) r.getMinLon(), + (int) r.getMinLon()}; + addWriter(new EdgeTreeWriter(lons, lats)); + return null; + } + + @Override + public Void visit(Point point) { + throw new UnsupportedOperationException("support for Point is a TODO"); + } + + @Override + public Void visit(MultiPoint multiPoint) { + throw new UnsupportedOperationException("support for MultiPoint is a TODO"); + } + + @Override + public Void visit(LinearRing ring) { + throw new IllegalArgumentException("invalid shape type found [Circle]"); + } + + @Override + public Void visit(Circle circle) { + throw new IllegalArgumentException("invalid shape type found [Circle]"); + } + + private int[] asIntArray(double[] doub) { + int[] intArr = new int[doub.length]; + for (int i = 0; i < intArr.length; i++) { + intArr[i] = (int) doub[i]; + } + return intArr; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java new file mode 100644 index 0000000000000..2d200e7f82e47 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -0,0 +1,97 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geo.geometry.LinearRing; +import org.elasticsearch.geo.geometry.Polygon; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.Collections; + +public class GeometryTreeTests extends ESTestCase { + + public void testRectangleShape() throws IOException { + for (int i = 0; i < 1000; i++) { + int minX = randomIntBetween(-180, 170); + int maxX = randomIntBetween(minX + 10, 180); + int minY = randomIntBetween(-90, 80); + int maxY = randomIntBetween(minY + 10, 90); + double[] x = new double[]{minX, maxX, maxX, minX, minX}; + double[] y = new double[]{minY, minY, maxY, maxY, minY}; + GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(y, x), Collections.emptyList())); + BytesRef bytes = writer.toBytesRef(); + GeometryTreeReader reader = new GeometryTreeReader(bytes); + + // box-query touches bottom-left corner + assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY)); + // box-query touches bottom-right corner + assertTrue(reader.containedInOrCrosses(maxX, minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), minY)); + // box-query touches top-right corner + assertTrue(reader.containedInOrCrosses(maxX, maxY, maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + // box-query touches top-left corner + assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), maxY, minX, maxY + randomIntBetween(1, 180))); + // box-query fully-enclosed inside rectangle + assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + (3 * maxY + minY) / 4)); + // box-query fully-contains poly + assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), + maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + // box-query half-in-half-out-right + assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + (3 * maxY + minY) / 4)); + // box-query half-in-half-out-left + assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + (3 * maxY + minY) / 4)); + // box-query half-in-half-out-top + assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + maxY + randomIntBetween(1, 1000))); + // box-query half-in-half-out-bottom + assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + + // box-query outside to the right + assertFalse(reader.containedInOrCrosses(maxX + randomIntBetween(1, 1000), minY, maxX + randomIntBetween(1001, 2000), maxY)); + // box-query outside to the left + assertFalse(reader.containedInOrCrosses(maxX - randomIntBetween(1001, 2000), minY, minX - randomIntBetween(1, 1000), maxY)); + // box-query outside to the top + assertFalse(reader.containedInOrCrosses(minX, maxY + randomIntBetween(1, 1000), maxX, maxY + randomIntBetween(1001, 2000))); + // box-query outside to the bottom + assertFalse(reader.containedInOrCrosses(minX, minY - randomIntBetween(1001, 2000), maxX, minY - randomIntBetween(1, 1000))); + } + } + + public void testPacMan() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // candidate containedInOrCrosses cell + int xMin = 2;//-5; + int xMax = 11;//0.000001; + int yMin = -1;//0; + int yMax = 1;//5; + + // test cell crossing poly + GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(py, px), Collections.emptyList())); + GeometryTreeReader reader = new GeometryTreeReader(writer.toBytesRef()); + assertTrue(reader.containedInOrCrosses(xMin, yMin, xMax, yMax)); + } +} From 6eeabc702e83d318c506eea3fcee053153d4d230 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 4 Jun 2019 12:23:48 -0700 Subject: [PATCH 03/62] fix changes --- .../org/elasticsearch/common/geo/EdgeTreeReader.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index 250e98e09d114..c902136ab9a42 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -24,7 +24,7 @@ import java.io.IOException; import java.nio.ByteBuffer; -import static org.apache.lucene.geo.GeoUtils.lineCrossesLine; +import static org.apache.lucene.geo.GeoUtils.lineCrossesLineWithBoundary; public class EdgeTreeReader { final BytesRef bytesRef; @@ -110,7 +110,7 @@ private boolean containsBottomLeft(ByteBufferStreamInput input, Edge root, int m if (root.maxY >= minY) { // is bbox-query contained within linearRing // cast infinite ray to the right from bottom-left of bbox-query to see if it intersects edge - if (lineCrossesLine(root.x1, root.y1, root.x2, root.y2,minX, minY, Integer.MAX_VALUE, minY)) { + if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2,minX, minY, Integer.MAX_VALUE, minY)) { res = true; } @@ -134,10 +134,10 @@ private boolean crosses(ByteBufferStreamInput input, Edge root, int minX, int mi if (root.maxY >= minY) { // does rectangle's edges intersect or reside inside polygon's edge - if (lineCrossesLine(root.x1, root.y1, root.x2, root.y2, minX, minY, maxX, minY) || - lineCrossesLine(root.x1, root.y1, root.x2, root.y2, maxX, minY, maxX, maxY) || - lineCrossesLine(root.x1, root.y1, root.x2, root.y2, maxX, maxY, minX, maxY) || - lineCrossesLine(root.x1, root.y1, root.x2, root.y2, minX, maxY, minX, minY)) { + if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, minX, minY, maxX, minY) || + lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, maxX, minY, maxX, maxY) || + lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, maxX, maxY, minX, maxY) || + lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, minX, maxY, minX, minY)) { return true; } From 49975f27635bb1b78c09daf024acf8f0a832df80 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 7 Jun 2019 13:25:39 -0700 Subject: [PATCH 04/62] make GeometryTree and EdgeTree writers implement Writeable (#42910) --- .../common/geo/EdgeTreeReader.java | 6 ++-- .../common/geo/EdgeTreeWriter.java | 24 +++++++--------- .../common/geo/GeometryTreeReader.java | 3 +- .../common/geo/GeometryTreeWriter.java | 28 +++++++++---------- .../common/geo/EdgeTreeTests.java | 19 +++++++++---- .../common/geo/GeometryTreeTests.java | 14 +++++++--- 6 files changed, 52 insertions(+), 42 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index c902136ab9a42..dc502f879aaf4 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -20,6 +20,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; import java.nio.ByteBuffer; @@ -29,8 +30,9 @@ public class EdgeTreeReader { final BytesRef bytesRef; - public EdgeTreeReader(BytesRef bytesRef) { - this.bytesRef = bytesRef; + public EdgeTreeReader(StreamInput input) throws IOException { + int treeBytesSize = input.readVInt(); + this.bytesRef = input.readBytesRef(treeBytesSize); } /** diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index 4ffba80a12817..96eaf296057b8 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -18,9 +18,8 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import java.io.IOException; import java.util.Arrays; @@ -28,7 +27,7 @@ /** * Shape edge-tree writer for use in doc-values */ -public class EdgeTreeWriter { +public class EdgeTreeWriter implements Writeable { /** * | minY | maxY | x1 | y1 | x2 | y2 | right_offset | @@ -68,17 +67,14 @@ public EdgeTreeWriter(int[] x, int[] y) { this.tree = createTree(edges, 0, edges.length - 1); } - public BytesRef toBytesRef() throws IOException { - BytesStreamOutput output = new BytesStreamOutput(4 * 4 + EDGE_SIZE_IN_BYTES * tree.size); - // write extent of edges - output.writeInt(minX); - output.writeInt(minY); - output.writeInt(maxX); - output.writeInt(maxY); - // write edge-tree itself - writeTree(tree, output); - output.close(); - return output.bytes().toBytesRef(); + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(4 * 4 + EDGE_SIZE_IN_BYTES * tree.size); + out.writeInt(minX); + out.writeInt(minY); + out.writeInt(maxX); + out.writeInt(maxY); + writeTree(tree, out); } private void writeTree(Edge edge, StreamOutput output) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index f709a3f20a598..53f78a2ebdba9 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -62,8 +62,7 @@ public boolean containedInOrCrosses(int minLon, int minLat, int maxLon, int maxL for (int i = 0; i < numTrees; i++) { ShapeType shapeType = input.readEnum(ShapeType.class); if (ShapeType.POLYGON.equals(shapeType)) { - BytesRef treeRef = input.readBytesRef(); - EdgeTreeReader reader = new EdgeTreeReader(treeRef); + EdgeTreeReader reader = new EdgeTreeReader(input); if (reader.containedInOrCrosses(minLon, minLat, maxLon, maxLat)) { return true; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 21de9b7f7601f..42bf9c3d951e3 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -18,8 +18,8 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.geo.geometry.Circle; import org.elasticsearch.geo.geometry.Geometry; import org.elasticsearch.geo.geometry.GeometryCollection; @@ -43,7 +43,7 @@ * appropriate tree structure for each type of * {@link Geometry} into a byte array. */ -public class GeometryTreeWriter { +public class GeometryTreeWriter implements Writeable { private final GeometryTreeBuilder builder; @@ -52,25 +52,23 @@ public class GeometryTreeWriter { geometry.visit(builder); } - public BytesRef toBytesRef() throws IOException { - BytesStreamOutput output = new BytesStreamOutput(); + @Override + public void writeTo(StreamOutput out) throws IOException { // only write a geometry extent for the tree if the tree // contains multiple sub-shapes boolean prependExtent = builder.shapeWriters.size() > 1; - output.writeBoolean(prependExtent); + out.writeBoolean(prependExtent); if (prependExtent) { - output.writeInt(builder.minLon); - output.writeInt(builder.minLat); - output.writeInt(builder.maxLon); - output.writeInt(builder.maxLat); + out.writeInt(builder.minLon); + out.writeInt(builder.minLat); + out.writeInt(builder.maxLon); + out.writeInt(builder.maxLat); } - output.writeVInt(builder.shapeWriters.size()); + out.writeVInt(builder.shapeWriters.size()); for (EdgeTreeWriter writer : builder.shapeWriters) { - output.writeEnum(ShapeType.POLYGON); - output.writeBytesRef(writer.toBytesRef()); + out.writeEnum(ShapeType.POLYGON); + writer.writeTo(out); } - output.close(); - return output.bytes().toBytesRef(); } class GeometryTreeBuilder implements GeometryVisitor { diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index 386c439c15527..38f2d6b669e8d 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -18,8 +18,9 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.geo.geometry.Polygon; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomShapeGenerator; @@ -38,8 +39,10 @@ public void testRectangleShape() throws IOException { int[] x = new int[]{minX, maxX, maxX, minX, minX}; int[] y = new int[]{minY, minY, maxY, maxY, minY}; EdgeTreeWriter writer = new EdgeTreeWriter(x, y); - BytesRef bytes = writer.toBytesRef(); - EdgeTreeReader reader = new EdgeTreeReader(bytes); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + EdgeTreeReader reader = new EdgeTreeReader(StreamInput.wrap(output.bytes().toBytesRef().bytes)); // box-query touches bottom-left corner assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY)); @@ -93,7 +96,10 @@ public void testSimplePolygon() throws IOException { int[] y = asIntArray(geo.getPolygon().getLats()); EdgeTreeWriter writer = new EdgeTreeWriter(x, y); - EdgeTreeReader reader = new EdgeTreeReader(writer.toBytesRef()); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + EdgeTreeReader reader = new EdgeTreeReader(StreamInput.wrap(output.bytes().toBytesRef().bytes)); // polygon fully contained within box assertTrue(reader.containedInOrCrosses(minXBox, minYBox, maxXBox, maxYBox)); // containedInOrCrosses @@ -121,7 +127,10 @@ public void testPacMan() throws Exception { // test cell crossing poly EdgeTreeWriter writer = new EdgeTreeWriter(px, py); - EdgeTreeReader reader = new EdgeTreeReader(writer.toBytesRef()); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + EdgeTreeReader reader = new EdgeTreeReader(StreamInput.wrap(output.bytes().toBytesRef().bytes)); assertTrue(reader.containsBottomLeft(xMin, yMin, xMax, yMax)); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 2d200e7f82e47..8df213820cd3d 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geo.geometry.LinearRing; import org.elasticsearch.geo.geometry.Polygon; import org.elasticsearch.test.ESTestCase; @@ -37,8 +37,11 @@ public void testRectangleShape() throws IOException { double[] x = new double[]{minX, maxX, maxX, minX, minX}; double[] y = new double[]{minY, minY, maxY, maxY, minY}; GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(y, x), Collections.emptyList())); - BytesRef bytes = writer.toBytesRef(); - GeometryTreeReader reader = new GeometryTreeReader(bytes); + + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); // box-query touches bottom-left corner assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY)); @@ -91,7 +94,10 @@ public void testPacMan() throws Exception { // test cell crossing poly GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(py, px), Collections.emptyList())); - GeometryTreeReader reader = new GeometryTreeReader(writer.toBytesRef()); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); assertTrue(reader.containedInOrCrosses(xMin, yMin, xMax, yMax)); } } From 92b24c3a0e2df4ff9883ea5f321ab5245ac3a17e Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 10 Jun 2019 14:05:04 -0700 Subject: [PATCH 05/62] Clean up Extent logic in Geo Tree readers (#42968) - min and max values of coordinates were difficult to track, this fixes that by introducing a new Extent object - Instead of re-wrapping ByteRef into a StreamInput, a stream input is made once - a new getExtent() method is introduced for use by aggregations like geo_bounds - re-use bounding-box containment checks --- .../common/geo/EdgeTreeReader.java | 146 ++++++++++-------- .../common/geo/EdgeTreeWriter.java | 89 ++++++----- .../org/elasticsearch/common/geo/Extent.java | 72 +++++++++ .../common/geo/GeometryTreeReader.java | 39 +++-- .../common/geo/GeometryTreeWriter.java | 8 +- .../common/geo/EdgeTreeTests.java | 14 +- .../common/geo/GeometryTreeTests.java | 4 + 7 files changed, 235 insertions(+), 137 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/Extent.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index dc502f879aaf4..b1f1eb5a1a622 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -18,71 +18,78 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; -import java.nio.ByteBuffer; +import java.util.Optional; import static org.apache.lucene.geo.GeoUtils.lineCrossesLineWithBoundary; public class EdgeTreeReader { - final BytesRef bytesRef; + final ByteBufferStreamInput input; + final int startPosition; - public EdgeTreeReader(StreamInput input) throws IOException { - int treeBytesSize = input.readVInt(); - this.bytesRef = input.readBytesRef(treeBytesSize); + public EdgeTreeReader(ByteBufferStreamInput input) throws IOException { + this.startPosition = input.position(); + this.input = input; + } + + public Extent getExtent() throws IOException { + resetInputPosition(); + return new Extent(input); } /** * Returns true if the rectangle query and the edge tree's shape overlap */ public boolean containedInOrCrosses(int minX, int minY, int maxX, int maxY) throws IOException { - return this.containsBottomLeft(minX, minY, maxX, maxY) || this.crosses(minX, minY, maxX, maxY); + Extent extent = new Extent(minX, minY, maxX, maxY); + return this.containsBottomLeft(extent) || this.crosses(extent); } - boolean containsBottomLeft(int minX, int minY, int maxX, int maxY) throws IOException { - ByteBufferStreamInput input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); - int thisMinX = input.readInt(); - int thisMinY = input.readInt(); - int thisMaxX = input.readInt(); - int thisMaxY = input.readInt(); + static Optional checkExtent(StreamInput input, Extent extent) throws IOException { + Extent edgeExtent = new Extent(input); - if (thisMinY > maxY || thisMaxX < minX || thisMaxY < minY || thisMinX > maxX) { - return false; // tree and bbox-query are disjoint + if (edgeExtent.minY > extent.maxY || edgeExtent.maxX < extent.minX + || edgeExtent.maxY < extent.minY || edgeExtent.minX > extent.maxX) { + return Optional.of(false); // tree and bbox-query are disjoint } - if (minX <= thisMinX && minY <= thisMinY && maxX >= thisMaxX && maxY >= thisMaxY) { - return true; // bbox-query fully contains tree's extent. + if (extent.minX <= edgeExtent.minX && extent.minY <= edgeExtent.minY + && extent.maxX >= edgeExtent.maxX && extent.maxY >= edgeExtent.maxY) { + return Optional.of(true); // bbox-query fully contains tree's extent. } - - return containsBottomLeft(input, readRoot(input, input.position()), minX, minY, maxX, maxY); + return Optional.empty(); } - public boolean crosses(int minX, int minY, int maxX, int maxY) throws IOException { - ByteBufferStreamInput input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); - int thisMinX = input.readInt(); - int thisMinY = input.readInt(); - int thisMaxX = input.readInt(); - int thisMaxY = input.readInt(); + boolean containsBottomLeft(Extent extent) throws IOException { + resetInputPosition(); - if (thisMinY > maxY || thisMaxX < minX || thisMaxY < minY || thisMinX > maxX) { - return false; // tree and bbox-query are disjoint + Optional extentCheck = checkExtent(input, extent); + if (extentCheck.isPresent()) { + return extentCheck.get(); } - if (minX <= thisMinX && minY <= thisMinY && maxX >= thisMaxX && maxY >= thisMaxY) { - return true; // bbox-query fully contains tree's extent. + return containsBottomLeft(readRoot(input.position()), extent); + } + + public boolean crosses(Extent extent) throws IOException { + resetInputPosition(); + + Optional extentCheck = checkExtent(input, extent); + if (extentCheck.isPresent()) { + return extentCheck.get(); } - return crosses(input, readRoot(input, input.position()), minX, minY, maxX, maxY); + return crosses(readRoot(input.position()), extent); } - public Edge readRoot(ByteBufferStreamInput input, int position) throws IOException { - return readEdge(input, position); + public Edge readRoot(int position) throws IOException { + return readEdge(position); } - private static Edge readEdge(ByteBufferStreamInput input, int position) throws IOException { + private Edge readEdge(int position) throws IOException { input.position(position); int minY = input.readInt(); int maxY = input.readInt(); @@ -95,33 +102,33 @@ private static Edge readEdge(ByteBufferStreamInput input, int position) throws I } - Edge readLeft(ByteBufferStreamInput input, Edge root) throws IOException { - return readEdge(input, root.streamOffset); + Edge readLeft(Edge root) throws IOException { + return readEdge(root.streamOffset); } - Edge readRight(ByteBufferStreamInput input, Edge root) throws IOException { - return readEdge(input, root.streamOffset + root.rightOffset); + Edge readRight(Edge root) throws IOException { + return readEdge(root.streamOffset + root.rightOffset); } /** * Returns true if the bottom-left point of the rectangle query is contained within the * tree's edges. */ - private boolean containsBottomLeft(ByteBufferStreamInput input, Edge root, int minX, int minY, int maxX, int maxY) throws IOException { + private boolean containsBottomLeft(Edge root, Extent extent) throws IOException { boolean res = false; - if (root.maxY >= minY) { + if (root.maxY >= extent.minY) { // is bbox-query contained within linearRing // cast infinite ray to the right from bottom-left of bbox-query to see if it intersects edge - if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2,minX, minY, Integer.MAX_VALUE, minY)) { + if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX, extent.minY, Integer.MAX_VALUE, extent.minY)) { res = true; } if (root.rightOffset > 0) { /* has left node */ - res ^= containsBottomLeft(input, readLeft(input, root), minX, minY, maxX, maxY); + res ^= containsBottomLeft(readLeft(root), extent); } - if (root.rightOffset > 0 && maxY >= root.minY) { /* no right node if rightOffset == -1 */ - res ^= containsBottomLeft(input, readRight(input, root), minX, minY, maxX, maxY); + if (root.rightOffset > 0 && extent.maxY >= root.minY) { /* no right node if rightOffset == -1 */ + res ^= containsBottomLeft(readRight(root), extent); } } return res; @@ -130,44 +137,47 @@ private boolean containsBottomLeft(ByteBufferStreamInput input, Edge root, int m /** * Returns true if the box crosses any edge in this edge subtree * */ - private boolean crosses(ByteBufferStreamInput input, Edge root, int minX, int minY, int maxX, int maxY) throws IOException { - boolean res = false; + private boolean crosses(Edge root, Extent extent) throws IOException { // we just have to cross one edge to answer the question, so we descend the tree and return when we do. - if (root.maxY >= minY) { + if (root.maxY >= extent.minY) { // does rectangle's edges intersect or reside inside polygon's edge - if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, minX, minY, maxX, minY) || - lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, maxX, minY, maxX, maxY) || - lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, maxX, maxY, minX, maxY) || - lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, minX, maxY, minX, minY)) { + if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + extent.minX, extent.minY, extent.maxX, extent.minY) || + lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + extent.maxX, extent.minY, extent.maxX, extent.maxY) || + lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + extent.maxX, extent.maxY, extent.minX, extent.maxY) || + lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + extent.minX, extent.maxY, extent.minX, extent.minY)) { return true; } - - if (root.rightOffset > 0) { /* has left node */ - if (crosses(input, readLeft(input, root), minX, minY, maxX, maxY)) { - return true; - } + /* has left node */ + if (root.rightOffset > 0 && crosses(readLeft(root), extent)) { + return true; } - if (root.rightOffset > 0 && maxY >= root.minY) { /* no right node if rightOffset == -1 */ - if (crosses(input, readRight(input, root), minX, minY, maxX, maxY)) { - return true; - } + /* no right node if rightOffset == -1 */ + if (root.rightOffset > 0 && extent.maxY >= root.minY && crosses(readRight(root), extent)) { + return true; } } return false; } + private void resetInputPosition() throws IOException { + input.position(startPosition); + } - private static class Edge { - int streamOffset; - int x1; - int y1; - int x2; - int y2; - int minY; - int maxY; - int rightOffset; + private static final class Edge { + final int streamOffset; + final int x1; + final int y1; + final int x2; + final int y2; + final int minY; + final int maxY; + final int rightOffset; /** * Object representing an edge node read from bytes diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index 96eaf296057b8..82a124775ba7f 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -34,69 +34,44 @@ public class EdgeTreeWriter implements Writeable { */ static final int EDGE_SIZE_IN_BYTES = 28; - int minX; - int minY; - int maxX; - int maxY; + Extent extent; final Edge tree; public EdgeTreeWriter(int[] x, int[] y) { - minX = minY = Integer.MAX_VALUE; - maxX = maxY = Integer.MIN_VALUE; + int minX = Integer.MAX_VALUE; + int minY = Integer.MAX_VALUE; + int maxX = Integer.MIN_VALUE; + int maxY = Integer.MIN_VALUE; Edge edges[] = new Edge[y.length - 1]; for (int i = 1; i < y.length; i++) { int y1 = y[i-1]; int x1 = x[i-1]; int y2 = y[i]; int x2 = x[i]; - int minY, maxY; + int edgeMinY, edgeMaxY; if (y1 < y2) { - minY = y1; - maxY = y2; + edgeMinY = y1; + edgeMaxY = y2; } else { - minY = y2; - maxY = y1; + edgeMinY = y2; + edgeMaxY = y1; } - edges[i - 1] = new Edge(x1, y1, x2, y2, minY, maxY); - this.minX = Math.min(this.minX, Math.min(x1, x2)); - this.minY = Math.min(this.minY, Math.min(y1, y2)); - this.maxX = Math.max(this.maxX, Math.max(x1, x2)); - this.maxY = Math.max(this.maxY, Math.max(y1, y2)); + edges[i - 1] = new Edge(x1, y1, x2, y2, edgeMinY, edgeMaxY); + minX = Math.min(minX, Math.min(x1, x2)); + minY = Math.min(minY, Math.min(y1, y2)); + maxX = Math.max(maxX, Math.max(x1, x2)); + maxY = Math.max(maxY, Math.max(y1, y2)); } Arrays.sort(edges); + this.extent = new Extent(minX, minY, maxX, maxY); this.tree = createTree(edges, 0, edges.length - 1); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(4 * 4 + EDGE_SIZE_IN_BYTES * tree.size); - out.writeInt(minX); - out.writeInt(minY); - out.writeInt(maxX); - out.writeInt(maxY); - writeTree(tree, out); - } - - private void writeTree(Edge edge, StreamOutput output) throws IOException { - if (edge == null) { - return; - } - output.writeInt(edge.minY); - output.writeInt(edge.maxY); - output.writeInt(edge.x1); - output.writeInt(edge.y1); - output.writeInt(edge.x2); - output.writeInt(edge.y2); - // left node is next node, write offset of right node - if (edge.left != null) { - output.writeInt(edge.left.size * EDGE_SIZE_IN_BYTES); - } else if (edge.right == null){ - output.writeInt(-1); - } else { - output.writeInt(0); - } - writeTree(edge.left, output); - writeTree(edge.right, output); + //out.writeVInt(4 * 4 + EDGE_SIZE_IN_BYTES * tree.size); + extent.writeTo(out); + tree.writeTo(out); } private static Edge createTree(Edge edges[], int low, int high) { @@ -126,7 +101,7 @@ private static Edge createTree(Edge edges[], int low, int high) { /** * Object representing an in-memory edge-tree to be serialized */ - static class Edge implements Comparable { + static class Edge implements Comparable, Writeable { final int x1; final int y1; final int x2; @@ -154,5 +129,29 @@ public int compareTo(Edge other) { } return ret; } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(minY); + out.writeInt(maxY); + out.writeInt(x1); + out.writeInt(y1); + out.writeInt(x2); + out.writeInt(y2); + // left node is next node, write offset of right node + if (left != null) { + out.writeInt(left.size * EDGE_SIZE_IN_BYTES); + } else if (right == null){ + out.writeInt(-1); + } else { + out.writeInt(0); + } + if (left != null) { + left.writeTo(out); + } + if (right != null) { + right.writeTo(out); + } + } } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java new file mode 100644 index 0000000000000..8e5e3b19d3c5e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -0,0 +1,72 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +import java.io.IOException; +import java.util.Objects; + +/** + * Object representing the extent of a geometry object within a + * {@link GeometryTreeWriter} and {@link EdgeTreeWriter}; + */ +final class Extent implements Writeable { + final int minX; + final int minY; + final int maxX; + final int maxY; + + Extent(int minX, int minY, int maxX, int maxY) { + this.minX = minX; + this.minY = minY; + this.maxX = maxX; + this.maxY = maxY; + } + + Extent(StreamInput input) throws IOException { + this(input.readInt(), input.readInt(), input.readInt(), input.readInt()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(minX); + out.writeInt(minY); + out.writeInt(maxX); + out.writeInt(maxY); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Extent extent = (Extent) o; + return minX == extent.minX && + minY == extent.minY && + maxX == extent.maxX && + maxY == extent.maxY; + } + + @Override + public int hashCode() { + return Objects.hash(minX, minY, maxX, maxY); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 53f78a2ebdba9..206e3b428be21 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -24,6 +24,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Optional; /** * A tree reader. @@ -33,28 +34,36 @@ */ public class GeometryTreeReader { - private final BytesRef bytesRef; + private final ByteBufferStreamInput input; public GeometryTreeReader(BytesRef bytesRef) { - this.bytesRef = bytesRef; + this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); } - public boolean containedInOrCrosses(int minLon, int minLat, int maxLon, int maxLat) throws IOException { - ByteBufferStreamInput input = new ByteBufferStreamInput( - ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + public Extent getExtent() throws IOException { + input.position(0); boolean hasExtent = input.readBoolean(); if (hasExtent) { - int thisMinLon = input.readInt(); - int thisMinLat = input.readInt(); - int thisMaxLon = input.readInt(); - int thisMaxLat = input.readInt(); - - if (thisMinLat > maxLat || thisMaxLon < minLon || thisMaxLat < minLat || thisMinLon > maxLon) { - return false; // tree and bbox-query are disjoint - } + return new Extent(input); + } + assert input.readVInt() == 1; + ShapeType shapeType = input.readEnum(ShapeType.class); + if (ShapeType.POLYGON.equals(shapeType)) { + EdgeTreeReader reader = new EdgeTreeReader(input); + return reader.getExtent(); + } else { + throw new UnsupportedOperationException("only polygons supported -- TODO"); + } + } - if (minLon <= thisMinLon && minLat <= thisMinLat && maxLon >= thisMaxLon && maxLat >= thisMaxLat) { - return true; // bbox-query fully contains tree's extent. + public boolean containedInOrCrosses(int minLon, int minLat, int maxLon, int maxLat) throws IOException { + input.position(0); + boolean hasExtent = input.readBoolean(); + if (hasExtent) { + Optional extentCheck = EdgeTreeReader.checkExtent(input, + new Extent(minLon, minLat, maxLon, maxLat)); + if (extentCheck.isPresent()) { + return extentCheck.get(); } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 42bf9c3d951e3..35b85d359bf14 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -87,10 +87,10 @@ class GeometryTreeBuilder implements GeometryVisitor { } private void addWriter(EdgeTreeWriter writer) { - minLon = Math.min(minLon, writer.minX); - minLat = Math.min(minLat, writer.minY); - maxLon = Math.max(maxLon, writer.maxX); - maxLat = Math.max(maxLat, writer.maxY); + minLon = Math.min(minLon, writer.extent.minX); + minLat = Math.min(minLat, writer.extent.minY); + maxLon = Math.max(maxLon, writer.extent.maxX); + maxLat = Math.max(maxLat, writer.extent.maxY); shapeWriters.add(writer); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index 38f2d6b669e8d..814c4b5736612 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -19,14 +19,17 @@ package org.elasticsearch.common.geo; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.geo.geometry.Polygon; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomShapeGenerator; import org.locationtech.spatial4j.shape.Rectangle; import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.hamcrest.Matchers.equalTo; public class EdgeTreeTests extends ESTestCase { @@ -42,7 +45,7 @@ public void testRectangleShape() throws IOException { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - EdgeTreeReader reader = new EdgeTreeReader(StreamInput.wrap(output.bytes().toBytesRef().bytes)); + EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); // box-query touches bottom-left corner assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY)); @@ -99,7 +102,8 @@ public void testSimplePolygon() throws IOException { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - EdgeTreeReader reader = new EdgeTreeReader(StreamInput.wrap(output.bytes().toBytesRef().bytes)); + EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + assertThat(reader.getExtent(), equalTo(new Extent(minXBox, minYBox, maxXBox, maxYBox))); // polygon fully contained within box assertTrue(reader.containedInOrCrosses(minXBox, minYBox, maxXBox, maxYBox)); // containedInOrCrosses @@ -130,8 +134,8 @@ public void testPacMan() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - EdgeTreeReader reader = new EdgeTreeReader(StreamInput.wrap(output.bytes().toBytesRef().bytes)); - assertTrue(reader.containsBottomLeft(xMin, yMin, xMax, yMax)); + EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + assertTrue(reader.containsBottomLeft(new Extent(xMin, yMin, xMax, yMax))); } private int[] asIntArray(double[] doub) { diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 8df213820cd3d..0a33149c7e989 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -26,6 +26,8 @@ import java.io.IOException; import java.util.Collections; +import static org.hamcrest.Matchers.equalTo; + public class GeometryTreeTests extends ESTestCase { public void testRectangleShape() throws IOException { @@ -43,6 +45,8 @@ public void testRectangleShape() throws IOException { output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + assertThat(reader.getExtent(), equalTo(new Extent(minX, minY, maxX, maxY))); + // box-query touches bottom-left corner assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY)); // box-query touches bottom-right corner From 2d06ac26c75ca72d5e2d5be63958b90430e43b2c Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 10 Jun 2019 16:30:15 -0700 Subject: [PATCH 06/62] use encoded lat and lon integers --- .../common/geo/EdgeTreeTests.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index 814c4b5736612..eb0e2603b57e9 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -28,6 +29,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; @@ -90,13 +92,13 @@ public void testSimplePolygon() throws IOException { ShapeBuilder builder = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON); Polygon geo = (Polygon) builder.buildGeometry(); Rectangle box = builder.buildS4J().getBoundingBox(); - int minXBox = (int) box.getMinX(); - int minYBox = (int) box.getMinY(); - int maxXBox = (int) box.getMaxX(); - int maxYBox = (int) box.getMaxY(); + int minXBox = GeoEncodingUtils.encodeLongitude(box.getMinX()); + int minYBox = GeoEncodingUtils.encodeLatitude(box.getMinY()); + int maxXBox = GeoEncodingUtils.encodeLongitude(box.getMaxX()); + int maxYBox = GeoEncodingUtils.encodeLatitude(box.getMaxY()); - int[] x = asIntArray(geo.getPolygon().getLons()); - int[] y = asIntArray(geo.getPolygon().getLats()); + int[] x = asIntArray(geo.getPolygon().getLons(), GeoEncodingUtils::encodeLongitude); + int[] y = asIntArray(geo.getPolygon().getLats(), GeoEncodingUtils::encodeLatitude); EdgeTreeWriter writer = new EdgeTreeWriter(x, y); BytesStreamOutput output = new BytesStreamOutput(); @@ -138,10 +140,10 @@ public void testPacMan() throws Exception { assertTrue(reader.containsBottomLeft(new Extent(xMin, yMin, xMax, yMax))); } - private int[] asIntArray(double[] doub) { + private int[] asIntArray(double[] doub, Function encode) { int[] intArr = new int[doub.length]; for (int i = 0; i < intArr.length; i++) { - intArr[i] = (int) doub[i]; + intArr[i] = encode.apply(doub[i]); } return intArr; } From 417c48c51e9467bedd217471e6b6e2e4702223ae Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 21 Jun 2019 16:31:15 -0700 Subject: [PATCH 07/62] Add GeometryTree support for point/multipoint (#43432) * Add GeometryTree support for point/multipoint This commit adds support for MultiPoint and Point shapes to be stored in GeometryTree. To represent the collection of points, a KDbush is used, which is a sorted array sorted recursively by alternating dimensions x/y. This work is inspired by https://github.com/mourner/kdbush The purpose of this reader is to check whether any subset of the points in the kd-tree are contained within the bounding-box query. * unify reader interface and cleanup multipoint usage * respond to review --- .../common/geo/EdgeTreeReader.java | 9 +- .../common/geo/EdgeTreeWriter.java | 15 +- .../common/geo/GeometryTreeReader.java | 36 ++-- .../common/geo/GeometryTreeWriter.java | 26 +-- .../common/geo/Point2DReader.java | 104 ++++++++++ .../common/geo/Point2DWriter.java | 178 ++++++++++++++++++ .../common/geo/ShapeTreeReader.java | 32 ++++ .../common/geo/ShapeTreeWriter.java | 32 ++++ .../common/geo/EdgeTreeTests.java | 52 ++--- .../common/geo/GeometryTreeTests.java | 79 +++++--- .../common/geo/Point2DTests.java | 80 ++++++++ .../geo/builders/PolygonBuilderTests.java | 1 + 12 files changed, 566 insertions(+), 78 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java create mode 100644 server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java create mode 100644 server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java create mode 100644 server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index b1f1eb5a1a622..883235c661c70 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -26,7 +26,11 @@ import static org.apache.lucene.geo.GeoUtils.lineCrossesLineWithBoundary; -public class EdgeTreeReader { +/** + * This {@link ShapeTreeReader} understands how to parse polygons + * serialized with the {@link EdgeTreeWriter} + */ +public class EdgeTreeReader implements ShapeTreeReader { final ByteBufferStreamInput input; final int startPosition; @@ -43,8 +47,7 @@ public Extent getExtent() throws IOException { /** * Returns true if the rectangle query and the edge tree's shape overlap */ - public boolean containedInOrCrosses(int minX, int minY, int maxX, int maxY) throws IOException { - Extent extent = new Extent(minX, minY, maxX, maxY); + public boolean intersects(Extent extent) throws IOException { return this.containsBottomLeft(extent) || this.crosses(extent); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index 82a124775ba7f..ccbc22dd57abe 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -20,6 +20,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.geo.geometry.ShapeType; import java.io.IOException; import java.util.Arrays; @@ -27,14 +28,14 @@ /** * Shape edge-tree writer for use in doc-values */ -public class EdgeTreeWriter implements Writeable { +public class EdgeTreeWriter extends ShapeTreeWriter { /** * | minY | maxY | x1 | y1 | x2 | y2 | right_offset | */ static final int EDGE_SIZE_IN_BYTES = 28; - Extent extent; + private final Extent extent; final Edge tree; public EdgeTreeWriter(int[] x, int[] y) { @@ -67,6 +68,16 @@ public EdgeTreeWriter(int[] x, int[] y) { this.tree = createTree(edges, 0, edges.length - 1); } + @Override + public Extent getExtent() { + return extent; + } + + @Override + public ShapeType getShapeType() { + return ShapeType.POLYGON; + } + @Override public void writeTo(StreamOutput out) throws IOException { //out.writeVInt(4 * 4 + EDGE_SIZE_IN_BYTES * tree.size); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 206e3b428be21..2b5c12a9debd1 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -48,20 +48,15 @@ public Extent getExtent() throws IOException { } assert input.readVInt() == 1; ShapeType shapeType = input.readEnum(ShapeType.class); - if (ShapeType.POLYGON.equals(shapeType)) { - EdgeTreeReader reader = new EdgeTreeReader(input); - return reader.getExtent(); - } else { - throw new UnsupportedOperationException("only polygons supported -- TODO"); - } + ShapeTreeReader reader = getReader(shapeType, input); + return reader.getExtent(); } - public boolean containedInOrCrosses(int minLon, int minLat, int maxLon, int maxLat) throws IOException { + public boolean intersects(Extent extent) throws IOException { input.position(0); boolean hasExtent = input.readBoolean(); if (hasExtent) { - Optional extentCheck = EdgeTreeReader.checkExtent(input, - new Extent(minLon, minLat, maxLon, maxLat)); + Optional extentCheck = EdgeTreeReader.checkExtent(input, extent); if (extentCheck.isPresent()) { return extentCheck.get(); } @@ -70,13 +65,26 @@ public boolean containedInOrCrosses(int minLon, int minLat, int maxLon, int maxL int numTrees = input.readVInt(); for (int i = 0; i < numTrees; i++) { ShapeType shapeType = input.readEnum(ShapeType.class); - if (ShapeType.POLYGON.equals(shapeType)) { - EdgeTreeReader reader = new EdgeTreeReader(input); - if (reader.containedInOrCrosses(minLon, minLat, maxLon, maxLat)) { - return true; - } + ShapeTreeReader reader = getReader(shapeType, input); + if (reader.intersects(extent)) { + return true; } } return false; } + + private static ShapeTreeReader getReader(ShapeType shapeType, ByteBufferStreamInput input) throws IOException { + switch (shapeType) { + case POLYGON: + return new EdgeTreeReader(input); + case POINT: + case MULTIPOINT: + return new Point2DReader(input); + case LINESTRING: + case MULTILINESTRING: + throw new UnsupportedOperationException("TODO: linestring and multilinestring"); + default: + throw new UnsupportedOperationException("unsupported shape type [" + shapeType + "]"); + } + } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 35b85d359bf14..32318595edfd9 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -32,7 +32,6 @@ import org.elasticsearch.geo.geometry.Point; import org.elasticsearch.geo.geometry.Polygon; import org.elasticsearch.geo.geometry.Rectangle; -import org.elasticsearch.geo.geometry.ShapeType; import java.io.IOException; import java.util.ArrayList; @@ -65,15 +64,15 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(builder.maxLat); } out.writeVInt(builder.shapeWriters.size()); - for (EdgeTreeWriter writer : builder.shapeWriters) { - out.writeEnum(ShapeType.POLYGON); + for (ShapeTreeWriter writer : builder.shapeWriters) { + out.writeEnum(writer.getShapeType()); writer.writeTo(out); } } class GeometryTreeBuilder implements GeometryVisitor { - private List shapeWriters; + private List shapeWriters; // integers are used to represent int-encoded lat/lon values int minLat; int maxLat; @@ -86,11 +85,12 @@ class GeometryTreeBuilder implements GeometryVisitor { maxLat = maxLon = Integer.MIN_VALUE; } - private void addWriter(EdgeTreeWriter writer) { - minLon = Math.min(minLon, writer.extent.minX); - minLat = Math.min(minLat, writer.extent.minY); - maxLon = Math.max(maxLon, writer.extent.maxX); - maxLat = Math.max(maxLat, writer.extent.maxY); + private void addWriter(ShapeTreeWriter writer) { + Extent extent = writer.getExtent(); + minLon = Math.min(minLon, extent.minX); + minLat = Math.min(minLat, extent.minY); + maxLon = Math.max(maxLon, extent.maxX); + maxLat = Math.max(maxLat, extent.maxY); shapeWriters.add(writer); } @@ -143,12 +143,16 @@ public Void visit(Rectangle r) { @Override public Void visit(Point point) { - throw new UnsupportedOperationException("support for Point is a TODO"); + Point2DWriter writer = new Point2DWriter(point); + addWriter(writer); + return null; } @Override public Void visit(MultiPoint multiPoint) { - throw new UnsupportedOperationException("support for MultiPoint is a TODO"); + Point2DWriter writer = new Point2DWriter(multiPoint); + addWriter(writer); + return null; } @Override diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java new file mode 100644 index 0000000000000..2db7f27025116 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java @@ -0,0 +1,104 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; + +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Deque; + +/** + * This {@link ShapeTreeReader} understands how to parse points + * serialized with the {@link Point2DWriter} + */ +class Point2DReader implements ShapeTreeReader { + private final ByteBufferStreamInput input; + private final int size; + private final int startPosition; + + Point2DReader(ByteBufferStreamInput input) throws IOException { + this.input = input; + this.size = input.readVInt(); + this.startPosition = input.position(); + } + + public Extent getExtent() throws IOException { + if (size == 2) { + int x = readX(0); + int y = readY(0); + return new Extent(x, y, x, y); + } else { + return new Extent(input); + } + } + + public boolean intersects(Extent extent) throws IOException { + Deque stack = new ArrayDeque<>(); + stack.push(0); + stack.push(size - 1); + stack.push(0); + while (stack.isEmpty() == false) { + int axis = stack.pop(); + int right = stack.pop(); + int left = stack.pop(); + + if (right - left <= Point2DWriter.LEAF_SIZE) { + for (int i = left; i <= right; i++) { + // TODO serialize to re-usable array instead of serializing in each step + int x = readX(i); + int y = readY(i); + if (x >= extent.minX && x <= extent.maxX && y >= extent.minY && y <= extent.maxY) { + return true; + } + } + continue; + } + + int middle = (right - left) >> 1; + int x = readX(middle); + int y = readY(middle); + if (x >= extent.minX && x <= extent.maxX && y >= extent.minY && y <= extent.maxY) { + return true; + } + if ((axis == 0 && extent.minX <= x) || (axis == 1 && extent.minY <= y)) { + stack.push(left); + stack.push(middle - 1); + stack.push(1 - axis); + } + if ((axis == 0 && extent.maxX >= x) || (axis == 1 && extent.maxY >= y)) { + stack.push(middle + 1); + stack.push(right); + stack.push(1 - axis); + } + } + + return false; + } + + private int readX(int pointIdx) throws IOException { + input.position(startPosition + 2 * pointIdx * Integer.BYTES); + return input.readInt(); + } + + private int readY(int pointIdx) throws IOException { + input.position(startPosition + (2 * pointIdx + 1) * Integer.BYTES); + return input.readInt(); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java new file mode 100644 index 0000000000000..d993a19dd4ea7 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java @@ -0,0 +1,178 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.geo.GeoEncodingUtils; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.geo.geometry.MultiPoint; +import org.elasticsearch.geo.geometry.Point; +import org.elasticsearch.geo.geometry.ShapeType; + +import java.io.IOException; + +/** + * points KD-Tree (2D) writer for use in doc-values. + * + * This work is influenced by https://github.com/mourner/kdbush (ISC licensed). + */ +public class Point2DWriter extends ShapeTreeWriter { + + private static final int K = 2; + private final Extent extent; + private final int[] coords; + // size of a leaf node where searches are done sequentially. + static final int LEAF_SIZE = 64; + + Point2DWriter(MultiPoint multiPoint) { + int numPoints = multiPoint.size(); + int minX = Integer.MAX_VALUE; + int minY = Integer.MAX_VALUE; + int maxX = Integer.MIN_VALUE; + int maxY = Integer.MIN_VALUE; + coords = new int[numPoints * K]; + int i = 0; + for (Point point : multiPoint) { + int x = GeoEncodingUtils.encodeLongitude(point.getLon()); + int y = GeoEncodingUtils.encodeLatitude(point.getLat()); + minX = Math.min(minX, x); + minY = Math.min(minY, y); + maxX = Math.max(maxX, x); + maxY = Math.max(maxY, y); + coords[2 * i] = x; + coords[2 * i + 1] = y; + i++; + } + sort(0, numPoints - 1, 0); + this.extent = new Extent(minX, minY, maxX, maxY); + } + + Point2DWriter(Point point) { + int x = GeoEncodingUtils.encodeLongitude(point.getLon()); + int y = GeoEncodingUtils.encodeLatitude(point.getLat()); + coords = new int[] {x, y}; + this.extent = new Extent(x, y, x, y); + } + + @Override + public Extent getExtent() { + return extent; + } + + @Override + public ShapeType getShapeType() { + return ShapeType.MULTIPOINT; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + int numPoints = coords.length >> 1; + out.writeVInt(numPoints); + if (numPoints > 1) { + extent.writeTo(out); + } + for (int coord : coords) { + out.writeInt(coord); + } + } + + private void sort(int left, int right, int depth) { + // since the reader will search through points within a leaf, + // there is no improved performance by sorting these points. + if (right - left <= LEAF_SIZE) { + return; + } + + int middle = (left + right) >> 1; + + select(left, right, middle, depth); + + sort(left, middle - 1, depth + 1); + sort(middle + 1, right, depth + 1); + } + + /** + * A slightly-modified Floyd-Rivest selection algorithm, + * https://en.wikipedia.org/wiki/Floyd%E2%80%93Rivest_algorithm + * + * @param left the index of the left point + * @param right the index of the right point + * @param k the pivot index + * @param depth the depth in the kd-tree + */ + private void select(int left, int right, int k, int depth) { + int axis = depth % K; + while (right > left) { + if (right - left > 600) { + double n = right - left + 1; + int i = k - left + 1; + double z = Math.log(n); + double s = 0.5 * Math.exp(2 * z / 3); + double sd = 0.5 * Math.sqrt(z * s * (n - s) / n) * ((i - n / 2) < 0 ? -1 : 1); + int newLeft = Math.max(left, (int) Math.floor(k - i * s / n + sd)); + int newRight = Math.min(right, (int) Math.floor(k + (n - i) * s / n + sd)); + select(newLeft, newRight, k, depth); + } + int t = coords[2 * k + axis]; + int i = left; + int j = right; + + swapPoint(i, j); + if (coords[2 * right + axis] > t) { + swapPoint(left, right); + } + + while (i < j) { + swapPoint(i, j); + i++; + j--; + while (coords[2 * i + axis] < t) { + i++; + } + while (coords[2 * j + axis] > t) { + j--; + } + } + + if (coords[2 * left + axis] == t) { + swapPoint(left, j); + } else { + j++; + swapPoint(j, right); + } + + if (j <= k) { + left = j + 1; + } + if (k <= j) { + right = j - 1; + } + } + } + + private void swapPoint(int i, int j) { + swap( 2 * i, 2 * j); + swap(2 * i + 1, 2 * j + 1); + } + + private void swap(int i, int j) { + int tmp = coords[i]; + coords[i] = coords[j]; + coords[j] = tmp; + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java new file mode 100644 index 0000000000000..6931007e76344 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.geo.geometry.Geometry; + +import java.io.IOException; + +/** + * Shape Reader to read different {@link Geometry} doc-values + */ +public interface ShapeTreeReader { + + Extent getExtent() throws IOException; + boolean intersects(Extent extent) throws IOException; +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java new file mode 100644 index 0000000000000..2c16377a9f719 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java @@ -0,0 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.geo.geometry.ShapeType; + +/** + * Shape writer for use in doc-values + */ +public abstract class ShapeTreeWriter implements Writeable { + + public abstract Extent getExtent(); + + public abstract ShapeType getShapeType(); +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index eb0e2603b57e9..abd94ae333fa6 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -50,40 +50,40 @@ public void testRectangleShape() throws IOException { EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); // box-query touches bottom-left corner - assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY)); + assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY))); // box-query touches bottom-right corner - assertTrue(reader.containedInOrCrosses(maxX, minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), minY)); + assertTrue(reader.intersects(new Extent(maxX, minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), minY))); // box-query touches top-right corner - assertTrue(reader.containedInOrCrosses(maxX, maxY, maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + assertTrue(reader.intersects(new Extent(maxX, maxY, maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180)))); // box-query touches top-left corner - assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), maxY, minX, maxY + randomIntBetween(1, 180))); + assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), maxY, minX, maxY + randomIntBetween(1, 180)))); // box-query fully-enclosed inside rectangle - assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, - (3 * maxY + minY) / 4)); + assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + (3 * maxY + minY) / 4))); // box-query fully-contains poly - assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), - maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), + maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180)))); // box-query half-in-half-out-right - assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), - (3 * maxY + minY) / 4)); + assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + (3 * maxY + minY) / 4))); // box-query half-in-half-out-left - assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, - (3 * maxY + minY) / 4)); + assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + (3 * maxY + minY) / 4))); // box-query half-in-half-out-top - assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), - maxY + randomIntBetween(1, 1000))); + assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + maxY + randomIntBetween(1, 1000)))); // box-query half-in-half-out-bottom - assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4))); // box-query outside to the right - assertFalse(reader.containedInOrCrosses(maxX + randomIntBetween(1, 1000), minY, maxX + randomIntBetween(1001, 2000), maxY)); + assertFalse(reader.intersects(new Extent(maxX + randomIntBetween(1, 1000), minY, maxX + randomIntBetween(1001, 2000), maxY))); // box-query outside to the left - assertFalse(reader.containedInOrCrosses(maxX - randomIntBetween(1001, 2000), minY, minX - randomIntBetween(1, 1000), maxY)); + assertFalse(reader.intersects(new Extent(maxX - randomIntBetween(1001, 2000), minY, minX - randomIntBetween(1, 1000), maxY))); // box-query outside to the top - assertFalse(reader.containedInOrCrosses(minX, maxY + randomIntBetween(1, 1000), maxX, maxY + randomIntBetween(1001, 2000))); + assertFalse(reader.intersects(new Extent(minX, maxY + randomIntBetween(1, 1000), maxX, maxY + randomIntBetween(1001, 2000)))); // box-query outside to the bottom - assertFalse(reader.containedInOrCrosses(minX, minY - randomIntBetween(1001, 2000), maxX, minY - randomIntBetween(1, 1000))); + assertFalse(reader.intersects(new Extent(minX, minY - randomIntBetween(1001, 2000), maxX, minY - randomIntBetween(1, 1000)))); } } @@ -107,16 +107,16 @@ public void testSimplePolygon() throws IOException { EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); assertThat(reader.getExtent(), equalTo(new Extent(minXBox, minYBox, maxXBox, maxYBox))); // polygon fully contained within box - assertTrue(reader.containedInOrCrosses(minXBox, minYBox, maxXBox, maxYBox)); - // containedInOrCrosses + assertTrue(reader.intersects(new Extent(minXBox, minYBox, maxXBox, maxYBox))); + // intersects if (maxYBox - 1 >= minYBox) { - assertTrue(reader.containedInOrCrosses(minXBox, minYBox, maxXBox, maxYBox - 1)); + assertTrue(reader.intersects(new Extent(minXBox, minYBox, maxXBox, maxYBox - 1))); } if (maxXBox -1 >= minXBox) { - assertTrue(reader.containedInOrCrosses(minXBox, minYBox, maxXBox - 1, maxYBox)); + assertTrue(reader.intersects(new Extent(minXBox, minYBox, maxXBox - 1, maxYBox))); } // does not cross - assertFalse(reader.containedInOrCrosses(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10)); + assertFalse(reader.intersects(new Extent(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10))); } } @@ -125,7 +125,7 @@ public void testPacMan() throws Exception { int[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; int[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - // candidate containedInOrCrosses cell + // candidate intersects cell int xMin = 2;//-5; int xMax = 11;//0.000001; int yMin = -1;//0; diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 0a33149c7e989..eafb836fb6586 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -20,11 +20,15 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geo.geometry.LinearRing; +import org.elasticsearch.geo.geometry.MultiPoint; +import org.elasticsearch.geo.geometry.Point; import org.elasticsearch.geo.geometry.Polygon; import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -48,40 +52,40 @@ public void testRectangleShape() throws IOException { assertThat(reader.getExtent(), equalTo(new Extent(minX, minY, maxX, maxY))); // box-query touches bottom-left corner - assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY)); + assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY))); // box-query touches bottom-right corner - assertTrue(reader.containedInOrCrosses(maxX, minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), minY)); + assertTrue(reader.intersects(new Extent(maxX, minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), minY))); // box-query touches top-right corner - assertTrue(reader.containedInOrCrosses(maxX, maxY, maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + assertTrue(reader.intersects(new Extent(maxX, maxY, maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180)))); // box-query touches top-left corner - assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), maxY, minX, maxY + randomIntBetween(1, 180))); + assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), maxY, minX, maxY + randomIntBetween(1, 180)))); // box-query fully-enclosed inside rectangle - assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, - (3 * maxY + minY) / 4)); + assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + (3 * maxY + minY) / 4))); // box-query fully-contains poly - assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), - maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), + maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180)))); // box-query half-in-half-out-right - assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), - (3 * maxY + minY) / 4)); + assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + (3 * maxY + minY) / 4))); // box-query half-in-half-out-left - assertTrue(reader.containedInOrCrosses(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, - (3 * maxY + minY) / 4)); + assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + (3 * maxY + minY) / 4))); // box-query half-in-half-out-top - assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), - maxY + randomIntBetween(1, 1000))); + assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + maxY + randomIntBetween(1, 1000)))); // box-query half-in-half-out-bottom - assertTrue(reader.containedInOrCrosses((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4))); // box-query outside to the right - assertFalse(reader.containedInOrCrosses(maxX + randomIntBetween(1, 1000), minY, maxX + randomIntBetween(1001, 2000), maxY)); + assertFalse(reader.intersects(new Extent(maxX + randomIntBetween(1, 1000), minY, maxX + randomIntBetween(1001, 2000), maxY))); // box-query outside to the left - assertFalse(reader.containedInOrCrosses(maxX - randomIntBetween(1001, 2000), minY, minX - randomIntBetween(1, 1000), maxY)); + assertFalse(reader.intersects(new Extent(maxX - randomIntBetween(1001, 2000), minY, minX - randomIntBetween(1, 1000), maxY))); // box-query outside to the top - assertFalse(reader.containedInOrCrosses(minX, maxY + randomIntBetween(1, 1000), maxX, maxY + randomIntBetween(1001, 2000))); + assertFalse(reader.intersects(new Extent(minX, maxY + randomIntBetween(1, 1000), maxX, maxY + randomIntBetween(1001, 2000)))); // box-query outside to the bottom - assertFalse(reader.containedInOrCrosses(minX, minY - randomIntBetween(1001, 2000), maxX, minY - randomIntBetween(1, 1000))); + assertFalse(reader.intersects(new Extent(minX, minY - randomIntBetween(1001, 2000), maxX, minY - randomIntBetween(1, 1000)))); } } @@ -90,7 +94,7 @@ public void testPacMan() throws Exception { double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - // candidate containedInOrCrosses cell + // candidate intersects cell int xMin = 2;//-5; int xMax = 11;//0.000001; int yMin = -1;//0; @@ -102,6 +106,37 @@ public void testPacMan() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.containedInOrCrosses(xMin, yMin, xMax, yMax)); + assertTrue(reader.intersects(new Extent(xMin, yMin, xMax, yMax))); + } + + public void testPacManPoints() throws Exception { + // pacman + List points = Arrays.asList( + new Point(0, 0), + new Point(5, 10), + new Point(9, 10), + new Point(10, 0), + new Point(9, -8), + new Point(0, -10), + new Point(-9, -8), + new Point(-10, 0), + new Point(-9, 10), + new Point(-5, 10) + ); + + + // candidate intersects cell + int xMin = 0; + int xMax = 11; + int yMin = -10; + int yMax = 9; + + // test cell crossing poly + GeometryTreeWriter writer = new GeometryTreeWriter(new MultiPoint(points)); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + assertTrue(reader.intersects(new Extent(xMin, yMin, xMax, yMax))); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java new file mode 100644 index 0000000000000..1a98704eff821 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.geo.GeoEncodingUtils; +import org.elasticsearch.common.geo.builders.PointBuilder; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.geometry.MultiPoint; +import org.elasticsearch.geo.geometry.Point; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.geo.RandomShapeGenerator; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +public class Point2DTests extends ESTestCase { + + public void testOnePoint() throws IOException { + PointBuilder gen = (PointBuilder) RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POINT); + Point point = gen.buildGeometry(); + int x = GeoEncodingUtils.encodeLongitude(point.getLon()); + int y = GeoEncodingUtils.encodeLatitude(point.getLat()); + Point2DWriter writer = new Point2DWriter(point); + + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + assertTrue(reader.intersects(new Extent(x, y, x, y))); + assertTrue(reader.intersects(new Extent(x, y, x + randomIntBetween(1, 10), y + randomIntBetween(1, 10)))); + assertTrue(reader.intersects(new Extent(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), x, y))); + assertTrue(reader.intersects(new Extent(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), + x + randomIntBetween(1, 10), y + randomIntBetween(1, 10)))); + assertFalse(reader.intersects(new Extent(x - randomIntBetween(10, 100), y - randomIntBetween(10, 100), + x - randomIntBetween(1, 10), y - randomIntBetween(1, 10)))); + } + + public void testPoints() throws IOException { + for (int i = 0; i < 100; i++) { + int minX = randomIntBetween(-180, 170); + int maxX = randomIntBetween(minX + 10, 180); + int minY = randomIntBetween(-90, 80); + int maxY = randomIntBetween(minY + 10, 90); + Extent extent = new Extent(GeoEncodingUtils.encodeLongitude(minX), GeoEncodingUtils.encodeLatitude(minY), + GeoEncodingUtils.encodeLongitude(maxX), GeoEncodingUtils.encodeLatitude(maxY)); + int numPoints = randomIntBetween(2, 1000); + + List points = new ArrayList<>(numPoints); + for (int j = 0; j < numPoints; j++) { + points.add(new Point(randomDoubleBetween(minY, maxY, true), randomDoubleBetween(minX, maxX, true))); + } + Point2DWriter writer = new Point2DWriter(new MultiPoint(points)); + + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + assertTrue(reader.intersects(extent)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java index 0d4f142785484..6095b0f772834 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/builders/PolygonBuilderTests.java @@ -33,6 +33,7 @@ public class PolygonBuilderTests extends AbstractShapeBuilderTestCase Date: Mon, 8 Jul 2019 10:25:57 -0700 Subject: [PATCH 08/62] fix bug in point2d select algorithm --- .../main/java/org/elasticsearch/common/geo/Point2DWriter.java | 2 +- .../test/java/org/elasticsearch/common/geo/Point2DTests.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java index d993a19dd4ea7..7e93dae4a57e7 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java @@ -132,7 +132,7 @@ private void select(int left, int right, int k, int depth) { int i = left; int j = right; - swapPoint(i, j); + swapPoint(left, k); if (coords[2 * right + axis] > t) { swapPoint(left, right); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java index 1a98704eff821..a2a504d266560 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java @@ -55,7 +55,7 @@ public void testOnePoint() throws IOException { } public void testPoints() throws IOException { - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 500; i++) { int minX = randomIntBetween(-180, 170); int maxX = randomIntBetween(minX + 10, 180); int minY = randomIntBetween(-90, 80); From bd3a93501b56bdda24a6f29d665b992d032cdd5d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 9 Jul 2019 12:16:07 -0700 Subject: [PATCH 09/62] fix point2d offbyone bug --- .../main/java/org/elasticsearch/common/geo/Point2DReader.java | 2 +- .../test/java/org/elasticsearch/common/geo/Point2DTests.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java index 2db7f27025116..207220e108122 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java @@ -40,7 +40,7 @@ class Point2DReader implements ShapeTreeReader { } public Extent getExtent() throws IOException { - if (size == 2) { + if (size == 1) { int x = readX(0); int y = readY(0); return new Extent(x, y, x, y); diff --git a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java index a2a504d266560..fb75d5a9f983b 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java @@ -32,6 +32,8 @@ import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.equalTo; + public class Point2DTests extends ESTestCase { public void testOnePoint() throws IOException { @@ -45,6 +47,7 @@ public void testOnePoint() throws IOException { writer.writeTo(output); output.close(); Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + assertThat(reader.getExtent(), equalTo(new Extent(x, y, x, y))); assertTrue(reader.intersects(new Extent(x, y, x, y))); assertTrue(reader.intersects(new Extent(x, y, x + randomIntBetween(1, 10), y + randomIntBetween(1, 10)))); assertTrue(reader.intersects(new Extent(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), x, y))); @@ -74,6 +77,7 @@ public void testPoints() throws IOException { writer.writeTo(output); output.close(); Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + assertThat(reader.getExtent(), equalTo(writer.getExtent())); assertTrue(reader.intersects(extent)); } } From 74c3f5df85fc2c74de0e9de46713136c839aecca Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 9 Jul 2019 12:43:23 -0700 Subject: [PATCH 10/62] add EdgeTree support for lines/multilines (#43949) The main change here is that edge-trees originally checked whether the queried extent could be contained within its shape. Since line-strings have no inner boundaries, this check is not useful, the line crosses check + extent-check-bounds is sufficient. --- .../common/geo/EdgeTreeReader.java | 14 ++-- .../common/geo/EdgeTreeWriter.java | 70 ++++++++++++------- .../common/geo/GeometryTreeReader.java | 4 +- .../common/geo/GeometryTreeWriter.java | 14 ++-- .../common/geo/EdgeTreeTests.java | 22 ++++-- .../common/geo/GeometryTreeTests.java | 48 ++++++++++--- 6 files changed, 122 insertions(+), 50 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index 883235c661c70..7da1aa66035fb 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -31,12 +31,14 @@ * serialized with the {@link EdgeTreeWriter} */ public class EdgeTreeReader implements ShapeTreeReader { - final ByteBufferStreamInput input; - final int startPosition; + private final ByteBufferStreamInput input; + private final int startPosition; + private final boolean hasArea; - public EdgeTreeReader(ByteBufferStreamInput input) throws IOException { + public EdgeTreeReader(ByteBufferStreamInput input, boolean hasArea) throws IOException { this.startPosition = input.position(); this.input = input; + this.hasArea = hasArea; } public Extent getExtent() throws IOException { @@ -48,7 +50,11 @@ public Extent getExtent() throws IOException { * Returns true if the rectangle query and the edge tree's shape overlap */ public boolean intersects(Extent extent) throws IOException { - return this.containsBottomLeft(extent) || this.crosses(extent); + if (hasArea) { + return containsBottomLeft(extent) || crosses(extent); + } else { + return crosses(extent); + } } static Optional checkExtent(StreamInput input, Extent extent) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index ccbc22dd57abe..90bdef95d2aac 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -20,10 +20,13 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.geo.geometry.Polygon; import org.elasticsearch.geo.geometry.ShapeType; import java.io.IOException; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; /** * Shape edge-tree writer for use in doc-values @@ -36,36 +39,52 @@ public class EdgeTreeWriter extends ShapeTreeWriter { static final int EDGE_SIZE_IN_BYTES = 28; private final Extent extent; + private final boolean hasArea; + private final int numShapes; final Edge tree; - public EdgeTreeWriter(int[] x, int[] y) { + + /** + * @param x array of the x-coordinate of points. + * @param y array of the y-coordinate of points. + * @param hasArea true if edge-tree represents a {@link Polygon} and has a non-zero area, false otherwise. + */ + EdgeTreeWriter(int[] x, int[] y, boolean hasArea) { + this(Collections.singletonList(x), Collections.singletonList(y), hasArea); + } + + EdgeTreeWriter(List x, List y, boolean hasArea) { + this.numShapes = x.size(); int minX = Integer.MAX_VALUE; int minY = Integer.MAX_VALUE; int maxX = Integer.MIN_VALUE; int maxY = Integer.MIN_VALUE; - Edge edges[] = new Edge[y.length - 1]; - for (int i = 1; i < y.length; i++) { - int y1 = y[i-1]; - int x1 = x[i-1]; - int y2 = y[i]; - int x2 = x[i]; - int edgeMinY, edgeMaxY; - if (y1 < y2) { - edgeMinY = y1; - edgeMaxY = y2; - } else { - edgeMinY = y2; - edgeMaxY = y1; + List edges = new ArrayList<>(); + for (int i = 0; i < y.size(); i++) { + for (int j = 1; j < y.get(i).length; j++) { + int y1 = y.get(i)[j - 1]; + int x1 = x.get(i)[j - 1]; + int y2 = y.get(i)[j]; + int x2 = x.get(i)[j]; + int edgeMinY, edgeMaxY; + if (y1 < y2) { + edgeMinY = y1; + edgeMaxY = y2; + } else { + edgeMinY = y2; + edgeMaxY = y1; + } + edges.add(new Edge(x1, y1, x2, y2, edgeMinY, edgeMaxY)); + minX = Math.min(minX, Math.min(x1, x2)); + minY = Math.min(minY, Math.min(y1, y2)); + maxX = Math.max(maxX, Math.max(x1, x2)); + maxY = Math.max(maxY, Math.max(y1, y2)); } - edges[i - 1] = new Edge(x1, y1, x2, y2, edgeMinY, edgeMaxY); - minX = Math.min(minX, Math.min(x1, x2)); - minY = Math.min(minY, Math.min(y1, y2)); - maxX = Math.max(maxX, Math.max(x1, x2)); - maxY = Math.max(maxY, Math.max(y1, y2)); } - Arrays.sort(edges); + edges.sort(Edge::compareTo); this.extent = new Extent(minX, minY, maxX, maxY); - this.tree = createTree(edges, 0, edges.length - 1); + this.tree = createTree(edges, 0, edges.size() - 1); + this.hasArea = hasArea; } @Override @@ -75,23 +94,22 @@ public Extent getExtent() { @Override public ShapeType getShapeType() { - return ShapeType.POLYGON; + return hasArea ? ShapeType.POLYGON : (numShapes > 1 ? ShapeType.MULTILINESTRING: ShapeType.LINESTRING); } @Override public void writeTo(StreamOutput out) throws IOException { - //out.writeVInt(4 * 4 + EDGE_SIZE_IN_BYTES * tree.size); extent.writeTo(out); tree.writeTo(out); } - private static Edge createTree(Edge edges[], int low, int high) { + private static Edge createTree(List edges, int low, int high) { if (low > high) { return null; } // add midpoint int mid = (low + high) >>> 1; - Edge newNode = edges[mid]; + Edge newNode = edges.get(mid); newNode.size = 1; // add children newNode.left = createTree(edges, low, mid - 1); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 2b5c12a9debd1..01c6eaec30beb 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -76,13 +76,13 @@ public boolean intersects(Extent extent) throws IOException { private static ShapeTreeReader getReader(ShapeType shapeType, ByteBufferStreamInput input) throws IOException { switch (shapeType) { case POLYGON: - return new EdgeTreeReader(input); + return new EdgeTreeReader(input, true); case POINT: case MULTIPOINT: return new Point2DReader(input); case LINESTRING: case MULTILINESTRING: - throw new UnsupportedOperationException("TODO: linestring and multilinestring"); + return new EdgeTreeReader(input, false); default: throw new UnsupportedOperationException("unsupported shape type [" + shapeType + "]"); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 32318595edfd9..3808d051ab932 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -104,14 +104,20 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - throw new UnsupportedOperationException("support for Line is a TODO"); + addWriter(new EdgeTreeWriter(asIntArray(line.getLons()), asIntArray(line.getLats()), false)); + return null; } @Override public Void visit(MultiLine multiLine) { + int size = multiLine.size(); + List x = new ArrayList<>(size); + List y = new ArrayList<>(size); for (Line line : multiLine) { - visit(line); + x.add(asIntArray(line.getLons())); + y.add(asIntArray(line.getLats())); } + addWriter(new EdgeTreeWriter(x, y, false)); return null; } @@ -119,7 +125,7 @@ public Void visit(MultiLine multiLine) { public Void visit(Polygon polygon) { // TODO (support holes) LinearRing outerShell = polygon.getPolygon(); - addWriter(new EdgeTreeWriter(asIntArray(outerShell.getLons()), asIntArray(outerShell.getLats()))); + addWriter(new EdgeTreeWriter(asIntArray(outerShell.getLons()), asIntArray(outerShell.getLats()), true)); return null; } @@ -137,7 +143,7 @@ public Void visit(Rectangle r) { (int) r.getMinLat()}; int[] lons = new int[] { (int) r.getMinLon(), (int) r.getMaxLon(), (int) r.getMaxLon(), (int) r.getMinLon(), (int) r.getMinLon()}; - addWriter(new EdgeTreeWriter(lons, lats)); + addWriter(new EdgeTreeWriter(lons, lats, true)); return null; } diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index abd94ae333fa6..ef3a7e1d5f268 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -23,12 +23,14 @@ import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geo.geometry.Polygon; +import org.elasticsearch.geo.geometry.ShapeType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomShapeGenerator; import org.locationtech.spatial4j.shape.Rectangle; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.List; import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; @@ -43,11 +45,11 @@ public void testRectangleShape() throws IOException { int maxY = randomIntBetween(minY + 10, 180); int[] x = new int[]{minX, maxX, maxX, minX, minX}; int[] y = new int[]{minY, minY, maxY, maxY, minY}; - EdgeTreeWriter writer = new EdgeTreeWriter(x, y); + EdgeTreeWriter writer = new EdgeTreeWriter(x, y, true); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); // box-query touches bottom-left corner assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY))); @@ -100,11 +102,11 @@ public void testSimplePolygon() throws IOException { int[] x = asIntArray(geo.getPolygon().getLons(), GeoEncodingUtils::encodeLongitude); int[] y = asIntArray(geo.getPolygon().getLats(), GeoEncodingUtils::encodeLatitude); - EdgeTreeWriter writer = new EdgeTreeWriter(x, y); + EdgeTreeWriter writer = new EdgeTreeWriter(x, y, true); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); assertThat(reader.getExtent(), equalTo(new Extent(minXBox, minYBox, maxXBox, maxYBox))); // polygon fully contained within box assertTrue(reader.intersects(new Extent(minXBox, minYBox, maxXBox, maxYBox))); @@ -132,14 +134,22 @@ public void testPacMan() throws Exception { int yMax = 1;//5; // test cell crossing poly - EdgeTreeWriter writer = new EdgeTreeWriter(px, py); + EdgeTreeWriter writer = new EdgeTreeWriter(px, py, true); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); assertTrue(reader.containsBottomLeft(new Extent(xMin, yMin, xMax, yMax))); } + public void testGetShapeType() { + int[] pointCoord = new int[] { 0 }; + assertThat(new EdgeTreeWriter(pointCoord, pointCoord, true).getShapeType(), equalTo(ShapeType.POLYGON)); + assertThat(new EdgeTreeWriter(pointCoord, pointCoord, false).getShapeType(), equalTo(ShapeType.LINESTRING)); + assertThat(new EdgeTreeWriter(List.of(pointCoord, pointCoord), List.of(pointCoord, pointCoord), false).getShapeType(), + equalTo(ShapeType.MULTILINESTRING)); + } + private int[] asIntArray(double[] doub, Function encode) { int[] intArr = new int[doub.length]; for (int i = 0; i < intArr.length; i++) { diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index eafb836fb6586..e61abdf75de82 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.geometry.Line; import org.elasticsearch.geo.geometry.LinearRing; import org.elasticsearch.geo.geometry.MultiPoint; import org.elasticsearch.geo.geometry.Point; @@ -89,24 +90,55 @@ public void testRectangleShape() throws IOException { } } - public void testPacMan() throws Exception { + public void testPacManPolygon() throws Exception { // pacman double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - // candidate intersects cell - int xMin = 2;//-5; - int xMax = 11;//0.000001; - int yMin = -1;//0; - int yMax = 1;//5; - // test cell crossing poly GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(py, px), Collections.emptyList())); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new Extent(xMin, yMin, xMax, yMax))); + assertTrue(reader.intersects(new Extent(2, -1, 11, 1))); + assertTrue(reader.intersects(new Extent(-12, -12, 12, 12))); + assertTrue(reader.intersects(new Extent(-2, -1, 2, 0))); + assertTrue(reader.intersects(new Extent(-5, -6, 2, -2))); + } + + public void testPacManClosedLineString() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // test cell crossing poly + GeometryTreeWriter writer = new GeometryTreeWriter(new Line(px, py)); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + assertTrue(reader.intersects(new Extent(2, -1, 11, 1))); + assertTrue(reader.intersects(new Extent(-12, -12, 12, 12))); + assertTrue(reader.intersects(new Extent(-2, -1, 2, 0))); + assertFalse(reader.intersects(new Extent(-5, -6, 2, -2))); + } + + public void testPacManLineString() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; + + // test cell crossing poly + GeometryTreeWriter writer = new GeometryTreeWriter(new Line(px, py)); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + assertTrue(reader.intersects(new Extent(2, -1, 11, 1))); + assertTrue(reader.intersects(new Extent(-12, -12, 12, 12))); + assertTrue(reader.intersects(new Extent(-2, -1, 2, 0))); + assertFalse(reader.intersects(new Extent(-5, -6, 2, -2))); } public void testPacManPoints() throws Exception { From ed052e38cc02d0e017fd6180c797e32938c22163 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 10 Jul 2019 11:38:35 -0700 Subject: [PATCH 11/62] wrap MultiGeoPointValues into generic type (#42863) To aid in keeping aggregation logic as simple as possible, the MultiGeoPointValues object that returns GeoPoint values for fields from doc-values is updated to return implementations of a geo-value object that can represent either points or shapes. --- .../expression/GeoEmptyValueSource.java | 8 +- .../expression/GeoLatitudeValueSource.java | 10 +- .../expression/GeoLongitudeValueSource.java | 10 +- .../org/elasticsearch/common/geo/Extent.java | 10 +- .../elasticsearch/common/geo/GeoUtils.java | 12 +- .../common/geo/GeometryTreeWriter.java | 2 +- ...FieldData.java => AtomicGeoFieldData.java} | 8 +- .../index/fielddata/FieldData.java | 43 ++-- .../fielddata/IndexGeoPointFieldData.java | 2 +- .../fielddata/IndexGeoShapeFieldData.java | 27 +++ .../index/fielddata/MultiGeoPointValues.java | 69 ------- .../index/fielddata/MultiGeoValues.java | 183 ++++++++++++++++++ .../index/fielddata/ScriptDocValues.java | 6 +- .../SingletonMultiGeoPointValues.java | 14 +- .../AbstractAtomicGeoPointFieldData.java | 16 +- .../AbstractAtomicGeoShapeFieldData.java | 66 +++++++ .../AbstractLatLonPointDVIndexFieldData.java | 6 +- .../AbstractLatLonShapeDVIndexFieldData.java | 89 +++++++++ .../plain/LatLonPointDVAtomicFieldData.java | 12 +- .../plain/LatLonShapeDVAtomicFieldData.java | 84 ++++++++ .../index/mapper/GeoShapeFieldMapper.java | 85 +++++++- .../functionscore/DecayFunctionBuilder.java | 10 +- .../bucket/geogrid/CellIdSource.java | 14 +- .../GeoDistanceRangeAggregatorFactory.java | 4 +- .../metrics/GeoBoundsAggregator.java | 8 +- .../metrics/GeoCentroidAggregator.java | 10 +- .../aggregations/support/MissingValues.java | 33 +++- .../aggregations/support/ValuesSource.java | 62 +++++- .../support/ValuesSourceConfig.java | 23 ++- .../support/ValuesSourceType.java | 3 +- .../search/sort/GeoDistanceSortBuilder.java | 4 +- .../AbstractGeoFieldDataTestCase.java | 8 +- .../index/fielddata/GeoFieldDataTests.java | 8 +- .../ScriptDocValuesGeoPointsTests.java | 12 +- .../support/MissingValuesTests.java | 17 +- .../support/ValuesSourceTypeTests.java | 4 + .../sql/analysis/index/IndexResolver.java | 6 +- 37 files changed, 767 insertions(+), 221 deletions(-) rename server/src/main/java/org/elasticsearch/index/fielddata/{AtomicGeoPointFieldData.java => AtomicGeoFieldData.java} (81%) create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java delete mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoShapeFieldData.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonShapeDVIndexFieldData.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java index 0b16aaf9dcde0..5eaf416e92d63 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoEmptyValueSource.java @@ -27,9 +27,9 @@ import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; /** * ValueSource to return non-zero if a field is missing. @@ -44,8 +44,8 @@ final class GeoEmptyValueSource extends ValueSource { @Override @SuppressWarnings("rawtypes") // ValueSource uses a rawtype public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException { - AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf); - final MultiGeoPointValues values = leafData.getGeoPointValues(); + AtomicGeoFieldData leafData = (AtomicGeoFieldData) fieldData.load(leaf); + final MultiGeoValues values = leafData.getGeoValues(); return new DoubleDocValues(this) { @Override public double doubleVal(int doc) throws IOException { diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java index fd812dac5a3a8..1722820f944e4 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLatitudeValueSource.java @@ -27,9 +27,9 @@ import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; /** * ValueSource to return latitudes as a double "stream" for geopoint fields @@ -44,13 +44,13 @@ final class GeoLatitudeValueSource extends ValueSource { @Override @SuppressWarnings("rawtypes") // ValueSource uses a rawtype public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException { - AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf); - final MultiGeoPointValues values = leafData.getGeoPointValues(); + AtomicGeoFieldData leafData = (AtomicGeoFieldData) fieldData.load(leaf); + final MultiGeoValues values = leafData.getGeoValues(); return new DoubleDocValues(this) { @Override public double doubleVal(int doc) throws IOException { if (values.advanceExact(doc)) { - return values.nextValue().getLat(); + return values.nextValue().lat(); } else { return 0.0; } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java index fd05d92d62350..9c27d90426c2c 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/GeoLongitudeValueSource.java @@ -27,9 +27,9 @@ import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; /** * ValueSource to return longitudes as a double "stream" for geopoint fields @@ -44,13 +44,13 @@ final class GeoLongitudeValueSource extends ValueSource { @Override @SuppressWarnings("rawtypes") // ValueSource uses a rawtype public FunctionValues getValues(Map context, LeafReaderContext leaf) throws IOException { - AtomicGeoPointFieldData leafData = (AtomicGeoPointFieldData) fieldData.load(leaf); - final MultiGeoPointValues values = leafData.getGeoPointValues(); + AtomicGeoFieldData leafData = (AtomicGeoFieldData) fieldData.load(leaf); + final MultiGeoValues values = leafData.getGeoValues(); return new DoubleDocValues(this) { @Override public double doubleVal(int doc) throws IOException { if (values.advanceExact(doc)) { - return values.nextValue().getLon(); + return values.nextValue().lon(); } else { return 0.0; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index 8e5e3b19d3c5e..5a3e7bf108c96 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -29,11 +29,11 @@ * Object representing the extent of a geometry object within a * {@link GeometryTreeWriter} and {@link EdgeTreeWriter}; */ -final class Extent implements Writeable { - final int minX; - final int minY; - final int maxX; - final int maxY; +public final class Extent implements Writeable { + public final int minX; + public final int minY; + public final int maxX; + public final int maxY; Extent(int minX, int minY, int maxX, int maxY) { this.minX = minX; diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java index f990a9750e0e1..cdb5cfa351741 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoUtils.java @@ -36,8 +36,7 @@ import org.elasticsearch.geo.geometry.Rectangle; import org.elasticsearch.geo.utils.Geohash; import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.GeoPointValues; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; @@ -646,10 +645,11 @@ public static double planeDistance(double lat1, double lon1, double lat2, double */ public static SortedNumericDoubleValues distanceValues(final GeoDistance distance, final DistanceUnit unit, - final MultiGeoPointValues geoPointValues, + final MultiGeoValues geoPointValues, final GeoPoint... fromPoints) { - final GeoPointValues singleValues = FieldData.unwrapSingleton(geoPointValues); + final MultiGeoValues singleValues = FieldData.unwrapSingleton(geoPointValues); if (singleValues != null && fromPoints.length == 1) { + assert singleValues.docValueCount() == 1; return FieldData.singleton(new NumericDoubleValues() { @Override @@ -660,7 +660,7 @@ public boolean advanceExact(int doc) throws IOException { @Override public double doubleValue() throws IOException { final GeoPoint from = fromPoints[0]; - final GeoPoint to = singleValues.geoPointValue(); + final MultiGeoValues.GeoValue to = singleValues.nextValue(); return distance.calculate(from.lat(), from.lon(), to.lat(), to.lon(), unit); } @@ -673,7 +673,7 @@ public boolean advanceExact(int target) throws IOException { resize(geoPointValues.docValueCount() * fromPoints.length); int v = 0; for (int i = 0; i < geoPointValues.docValueCount(); ++i) { - final GeoPoint point = geoPointValues.nextValue(); + final MultiGeoValues.GeoValue point = geoPointValues.nextValue(); for (GeoPoint from : fromPoints) { values[v] = distance.calculate(from.lat(), from.lon(), point.lat(), point.lon(), unit); v++; diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 3808d051ab932..23447b0db9ede 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -46,7 +46,7 @@ public class GeometryTreeWriter implements Writeable { private final GeometryTreeBuilder builder; - GeometryTreeWriter(Geometry geometry) { + public GeometryTreeWriter(Geometry geometry) { builder = new GeometryTreeBuilder(); geometry.visit(builder); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoFieldData.java similarity index 81% rename from server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java rename to server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoFieldData.java index a496e1d93ecd3..0dfd9c7fe3515 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoPointFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/AtomicGeoFieldData.java @@ -20,13 +20,13 @@ /** - * {@link AtomicFieldData} specialization for geo points. + * {@link AtomicFieldData} specialization for geo points and shapes. */ -public interface AtomicGeoPointFieldData extends AtomicFieldData { +public interface AtomicGeoFieldData extends AtomicFieldData { /** - * Return geo-point values. + * Return geo values. */ - MultiGeoPointValues getGeoPointValues(); + MultiGeoValues getGeoValues(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index 68b8f2c85325f..e2e3cade58558 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.geo.GeoPoint; import java.io.IOException; import java.util.ArrayList; @@ -69,27 +68,28 @@ public static SortedNumericDoubleValues emptySortedNumericDoubles() { return singleton(emptyNumericDouble()); } - public static GeoPointValues emptyGeoPoint() { - return new GeoPointValues() { + /** + * Return a {@link MultiGeoValues} that doesn't contain any value. + */ + public static MultiGeoValues emptyMultiGeoValues() { + return new MultiGeoValues() { @Override - public boolean advanceExact(int doc) throws IOException { + public boolean advanceExact(int doc) { return false; } @Override - public GeoPoint geoPointValue() { + public int docValueCount() { + return 0; + } + + @Override + public GeoValue nextValue() { throw new UnsupportedOperationException(); } }; } - /** - * Return a {@link SortedNumericDoubleValues} that doesn't contain any value. - */ - public static MultiGeoPointValues emptyMultiGeoPoints() { - return singleton(emptyGeoPoint()); - } - /** * Returns a {@link DocValueBits} representing all documents from values that have a value. */ @@ -119,7 +119,7 @@ public boolean advanceExact(int doc) throws IOException { * Returns a {@link DocValueBits} representing all documents from pointValues that have * a value. */ - public static DocValueBits docsWithValue(final MultiGeoPointValues pointValues) { + public static DocValueBits docsWithValue(final MultiGeoValues pointValues) { return new DocValueBits() { @Override public boolean advanceExact(int doc) throws IOException { @@ -246,20 +246,13 @@ public static NumericDoubleValues unwrapSingleton(SortedNumericDoubleValues valu } /** - * Returns a multi-valued view over the provided {@link GeoPointValues}. - */ - public static MultiGeoPointValues singleton(GeoPointValues values) { - return new SingletonMultiGeoPointValues(values); - } - - /** - * Returns a single-valued view of the {@link MultiGeoPointValues}, - * if it was previously wrapped with {@link #singleton(GeoPointValues)}, + * Returns a single-valued view of the {@link MultiGeoValues}, + * if it was previously wrapped {@link SingletonMultiGeoPointValues}, * or null. */ - public static GeoPointValues unwrapSingleton(MultiGeoPointValues values) { + public static MultiGeoValues unwrapSingleton(MultiGeoValues values) { if (values instanceof SingletonMultiGeoPointValues) { - return ((SingletonMultiGeoPointValues) values).getGeoPointValues(); + return ((SingletonMultiGeoPointValues) values).getGeoValues(); } return null; } @@ -375,7 +368,7 @@ public BytesRef nextValue() throws IOException { * typically used for scripts or for the `map` execution mode of terms aggs. * NOTE: this is very slow! */ - public static SortedBinaryDocValues toString(final MultiGeoPointValues values) { + public static SortedBinaryDocValues toString(final MultiGeoValues values) { return toString(new ToStringValues() { @Override public boolean advanceExact(int doc) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java index 53466e9b4ec61..959e3b9ffea26 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java @@ -23,5 +23,5 @@ /** * Specialization of {@link IndexFieldData} for geo points. */ -public interface IndexGeoPointFieldData extends IndexFieldData { +public interface IndexGeoPointFieldData extends IndexFieldData { } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java new file mode 100644 index 0000000000000..841fd1e91d655 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + + +/** + * Specialization of {@link IndexFieldData} for geo points. + */ +public interface IndexGeoShapeFieldData extends IndexFieldData { +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java deleted file mode 100644 index c80c337c6d0e3..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoPointValues.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.index.fielddata; - -import org.elasticsearch.common.geo.GeoPoint; - -import java.io.IOException; - -/** - * A stateful lightweight per document set of {@link GeoPoint} values. - * To iterate over values in a document use the following pattern: - * 
- *   GeoPointValues values = ..;
- *   values.setDocId(docId);
- *   final int numValues = values.count();
- *   for (int i = 0; i < numValues; i++) {
- *       GeoPoint value = values.valueAt(i);
- *       // process value
- *   }
- *
- * The set of values associated with a document might contain duplicates and - * comes in a non-specified order. - */ -public abstract class MultiGeoPointValues { - - /** - * Creates a new {@link MultiGeoPointValues} instance - */ - protected MultiGeoPointValues() { - } - - /** - * Advance this instance to the given document id - * @return true if there is a value for this document - */ - public abstract boolean advanceExact(int doc) throws IOException; - - /** - * Return the number of geo points the current document has. - */ - public abstract int docValueCount(); - - /** - * Return the next value associated with the current document. This must not be - * called more than {@link #docValueCount()} times. - * - * Note: the returned {@link GeoPoint} might be shared across invocations. - * - * @return the next value for the current docID set to {@link #advanceExact(int)}. - */ - public abstract GeoPoint nextValue() throws IOException; - -} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java new file mode 100644 index 0000000000000..958e79a071ea9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -0,0 +1,183 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.geo.GeoEncodingUtils; +import org.elasticsearch.common.geo.Extent; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeometryTreeReader; + +import java.io.IOException; + +/** + * A stateful lightweight per document set of geo values. + * To iterate over values in a document use the following pattern: + * 
+ *   MultiGeoValues values = ..;
+ *   values.setDocId(docId);
+ *   final int numValues = values.count();
+ *   for (int i = 0; i < numValues; i++) {
+ *       GeoValue value = values.valueAt(i);
+ *       // process value
+ *   }
+ *
+ * The set of values associated with a document might contain duplicates and + * comes in a non-specified order. + */ +public abstract class MultiGeoValues { + + /** + * Creates a new {@link MultiGeoValues} instance + */ + protected MultiGeoValues() { + } + + /** + * Advance this instance to the given document id + * @return true if there is a value for this document + */ + public abstract boolean advanceExact(int doc) throws IOException; + + /** + * Return the number of geo points the current document has. + */ + public abstract int docValueCount(); + + /** + * Return the next value associated with the current document. This must not be + * called more than {@link #docValueCount()} times. + * + * Note: the returned {@link GeoValue} might be shared across invocations. + * + * @return the next value for the current docID set to {@link #advanceExact(int)}. + */ + public abstract GeoValue nextValue() throws IOException; + + public static class GeoPointValue implements GeoValue { + private final GeoPoint geoPoint; + + public GeoPointValue(GeoPoint geoPoint) { + this.geoPoint = geoPoint; + } + + public GeoPoint geoPoint() { + return geoPoint; + } + + @Override + public double minLat() { + return geoPoint.lat(); + } + + @Override + public double maxLat() { + return geoPoint.lat(); + } + + @Override + public double minLon() { + return geoPoint.lon(); + } + + @Override + public double maxLon() { + return geoPoint.lon(); + } + + @Override + public double lat() { + return geoPoint.lat(); + } + + @Override + public double lon() { + return geoPoint.lon(); + } + + @Override + public String toString() { + return geoPoint.toString(); + } + } + + public static class GeoShapeValue implements GeoValue { + private final double minLat; + private final double maxLat; + private final double minLon; + private final double maxLon; + + public GeoShapeValue(GeometryTreeReader reader) throws IOException { + Extent extent = reader.getExtent(); + this.minLat = GeoEncodingUtils.decodeLatitude(extent.minY); + this.maxLat = GeoEncodingUtils.decodeLatitude(extent.maxY); + this.maxLon = GeoEncodingUtils.decodeLongitude(extent.maxX); + this.minLon = GeoEncodingUtils.decodeLongitude(extent.minX); + } + + public GeoShapeValue(double minLat, double minLon, double maxLat, double maxLon) { + this.minLat = minLat; + this.minLon = minLon; + this.maxLat = maxLat; + this.maxLon = maxLon; + } + + @Override + public double minLat() { + return minLat; + } + + @Override + public double maxLat() { + return maxLat; + } + + @Override + public double minLon() { + return minLon; + } + + @Override + public double maxLon() { + return maxLon; + } + + @Override + public double lat() { + throw new UnsupportedOperationException("centroid of GeoShape is not defined"); + } + + @Override + public double lon() { + throw new UnsupportedOperationException("centroid of GeoShape is not defined"); + } + } + + /** + * interface for geo-shape and geo-point doc-values to + * retrieve properties used in aggregations. + */ + public interface GeoValue { + double minLat(); + double maxLat(); + double minLon(); + double maxLon(); + double lat(); + double lon(); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java index 69f48a74c13da..249e1609e4666 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java @@ -260,11 +260,11 @@ public int size() { public static final class GeoPoints extends ScriptDocValues { - private final MultiGeoPointValues in; + private final MultiGeoValues in; private GeoPoint[] values = new GeoPoint[0]; private int count; - public GeoPoints(MultiGeoPointValues in) { + public GeoPoints(MultiGeoValues in) { this.in = in; } @@ -273,7 +273,7 @@ public void setNextDocId(int docId) throws IOException { if (in.advanceExact(docId)) { resize(in.docValueCount()); for (int i = 0; i < count; i++) { - GeoPoint point = in.nextValue(); + MultiGeoValues.GeoValue point = in.nextValue(); values[i] = new GeoPoint(point.lat(), point.lon()); } } else { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java index bae522e7b5081..2e56a42e1604b 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java @@ -19,15 +19,13 @@ package org.elasticsearch.index.fielddata; -import org.elasticsearch.common.geo.GeoPoint; - import java.io.IOException; -final class SingletonMultiGeoPointValues extends MultiGeoPointValues { +final class SingletonMultiGeoPointValues extends MultiGeoValues { - private final GeoPointValues in; + private final MultiGeoValues in; - SingletonMultiGeoPointValues(GeoPointValues in) { + SingletonMultiGeoPointValues(MultiGeoValues in) { this.in = in; } @@ -42,11 +40,11 @@ public int docValueCount() { } @Override - public GeoPoint nextValue() { - return in.geoPointValue(); + public GeoValue nextValue() throws IOException { + return in.nextValue(); } - GeoPointValues getGeoPointValues() { + MultiGeoValues getGeoValues() { return in; } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java index 5d6575e43783c..3767d795e3547 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoPointFieldData.java @@ -19,28 +19,28 @@ package org.elasticsearch.index.fielddata.plain; import org.apache.lucene.util.Accountable; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.FieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import java.util.Collection; import java.util.Collections; -public abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoPointFieldData { +public abstract class AbstractAtomicGeoPointFieldData implements AtomicGeoFieldData { @Override public final SortedBinaryDocValues getBytesValues() { - return FieldData.toString(getGeoPointValues()); + return FieldData.toString(getGeoValues()); } @Override public final ScriptDocValues.GeoPoints getScriptValues() { - return new ScriptDocValues.GeoPoints(getGeoPointValues()); + return new ScriptDocValues.GeoPoints(getGeoValues()); } - public static AtomicGeoPointFieldData empty(final int maxDoc) { + public static AtomicGeoFieldData empty(final int maxDoc) { return new AbstractAtomicGeoPointFieldData() { @Override @@ -58,8 +58,8 @@ public void close() { } @Override - public MultiGeoPointValues getGeoPointValues() { - return FieldData.emptyMultiGeoPoints(); + public MultiGeoValues getGeoValues() { + return FieldData.emptyMultiGeoValues(); } }; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoShapeFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoShapeFieldData.java new file mode 100644 index 0000000000000..bb98623e84697 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractAtomicGeoShapeFieldData.java @@ -0,0 +1,66 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.util.Accountable; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; +import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; + +import java.util.Collection; +import java.util.Collections; + +public abstract class AbstractAtomicGeoShapeFieldData implements AtomicGeoFieldData { + + @Override + public final SortedBinaryDocValues getBytesValues() { + return FieldData.toString(getGeoValues()); + } + + @Override + public final ScriptDocValues.BytesRefs getScriptValues() { + throw new UnsupportedOperationException(); + } + + public static AtomicGeoFieldData empty(final int maxDoc) { + return new AbstractAtomicGeoShapeFieldData() { + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() { + } + + @Override + public MultiGeoValues getGeoValues() { + return FieldData.emptyMultiGeoValues(); + } + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java index ed77d3d5f8b37..7cf9e6df3daea 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonPointDVIndexFieldData.java @@ -27,7 +27,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.fielddata.AtomicGeoPointFieldData; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; @@ -54,7 +54,7 @@ public LatLonPointDVIndexFieldData(Index index, String fieldName) { } @Override - public AtomicGeoPointFieldData load(LeafReaderContext context) { + public AtomicGeoFieldData load(LeafReaderContext context) { LeafReader reader = context.reader(); FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName); if (info != null) { @@ -64,7 +64,7 @@ public AtomicGeoPointFieldData load(LeafReaderContext context) { } @Override - public AtomicGeoPointFieldData loadDirect(LeafReaderContext context) throws Exception { + public AtomicGeoFieldData loadDirect(LeafReaderContext context) throws Exception { return load(context); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonShapeDVIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonShapeDVIndexFieldData.java new file mode 100644 index 0000000000000..4239ffd5204f9 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractLatLonShapeDVIndexFieldData.java @@ -0,0 +1,89 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.AtomicGeoFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.MultiValueMode; + +public abstract class AbstractLatLonShapeDVIndexFieldData extends DocValuesIndexFieldData implements IndexGeoShapeFieldData { + AbstractLatLonShapeDVIndexFieldData(Index index, String fieldName) { + super(index, fieldName); + } + + @Override + public SortField sortField(@Nullable Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, + boolean reverse) { + throw new IllegalArgumentException("can't sort on geo_shape field without using specific sorting feature, like geo_distance"); + } + + public static class LatLonShapeDVIndexFieldData extends AbstractLatLonShapeDVIndexFieldData { + public LatLonShapeDVIndexFieldData(Index index, String fieldName) { + super(index, fieldName); + } + + @Override + public AtomicGeoFieldData load(LeafReaderContext context) { + LeafReader reader = context.reader(); + FieldInfo info = reader.getFieldInfos().fieldInfo(fieldName); + if (info != null) { + checkCompatible(info); + } + return new LatLonShapeDVAtomicFieldData(reader, fieldName); + } + + @Override + public AtomicGeoFieldData loadDirect(LeafReaderContext context) throws Exception { + return load(context); + } + + /** helper: checks a fieldinfo and throws exception if its definitely not a LatLonDocValuesField */ + static void checkCompatible(FieldInfo fieldInfo) { + // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment. + if (fieldInfo.getDocValuesType() != DocValuesType.NONE + && fieldInfo.getDocValuesType() != DocValuesType.BINARY) { + throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + + fieldInfo.getDocValuesType() + " but this type has docValuesType=" + + DocValuesType.BINARY + ", is the field really a geo-shape field?"); + } + } + } + + public static class Builder implements IndexFieldData.Builder { + @Override + public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, + CircuitBreakerService breakerService, MapperService mapperService) { + // ignore breaker + return new LatLonShapeDVIndexFieldData(indexSettings.getIndex(), fieldType.name()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java index dba7dfb0c9e99..b2afd1ad045b7 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java @@ -24,7 +24,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.Accountable; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import java.io.IOException; import java.util.Collection; @@ -56,12 +56,12 @@ public void close() { } @Override - public MultiGeoPointValues getGeoPointValues() { + public MultiGeoValues getGeoValues() { try { final SortedNumericDocValues numericValues = DocValues.getSortedNumeric(reader, fieldName); - return new MultiGeoPointValues() { + return new MultiGeoValues() { - final GeoPoint point = new GeoPoint(); + final GeoPointValue point = new GeoPointValue(new GeoPoint()); @Override public boolean advanceExact(int doc) throws IOException { @@ -74,9 +74,9 @@ public int docValueCount() { } @Override - public GeoPoint nextValue() throws IOException { + public GeoValue nextValue() throws IOException { final long encoded = numericValues.nextValue(); - point.reset(GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)), + point.geoPoint().reset(GeoEncodingUtils.decodeLatitude((int) (encoded >>> 32)), GeoEncodingUtils.decodeLongitude((int) encoded)); return point; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java new file mode 100644 index 0000000000000..ec03a3959d482 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java @@ -0,0 +1,84 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.fielddata.plain; + +import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.util.Accountable; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeometryTreeReader; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; + +final class LatLonShapeDVAtomicFieldData extends AbstractAtomicGeoShapeFieldData { + private final LeafReader reader; + private final String fieldName; + + LatLonShapeDVAtomicFieldData(LeafReader reader, String fieldName) { + super(); + this.reader = reader; + this.fieldName = fieldName; + } + + @Override + public long ramBytesUsed() { + return 0; // not exposed by lucene + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + @Override + public void close() { + // noop + } + + @Override + public MultiGeoValues getGeoValues() { + try { + final BinaryDocValues binaryValues = DocValues.getBinary(reader, fieldName); + return new MultiGeoValues() { + + @Override + public boolean advanceExact(int doc) throws IOException { + return binaryValues.advanceExact(doc); + } + + @Override + public int docValueCount() { + return 1; + } + + @Override + public GeoValue nextValue() throws IOException { + final BytesRef encoded = binaryValues.binaryValue(); + return new GeoShapeValue(new GeometryTreeReader(encoded)); + } + }; + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 6449c06fbe1ad..ebdb08d5040df 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -23,9 +23,18 @@ import org.apache.lucene.geo.Line; import org.apache.lucene.geo.Polygon; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.geo.GeometryTreeWriter; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.parsers.ShapeParser; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.geo.geometry.Circle; import org.elasticsearch.geo.geometry.Geometry; @@ -36,10 +45,15 @@ import org.elasticsearch.geo.geometry.MultiPoint; import org.elasticsearch.geo.geometry.MultiPolygon; import org.elasticsearch.geo.geometry.Point; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.plain.AbstractLatLonShapeDVIndexFieldData; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.index.query.QueryShardException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** * FieldMapper for indexing {@link LatLonShape}s. @@ -74,6 +88,11 @@ public GeoShapeFieldMapper build(BuilderContext context) { return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } + + @Override + public boolean defaultDocValues(Version indexCreated) { + return Version.V_8_0_0.onOrBefore(indexCreated); + } } public static final class GeoShapeFieldType extends BaseGeoShapeFieldType { @@ -85,6 +104,26 @@ protected GeoShapeFieldType(GeoShapeFieldType ref) { super(ref); } + public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + failIfNoDocValues(); + return new AbstractLatLonShapeDVIndexFieldData.Builder(); + } + + @Override + public Query existsQuery(QueryShardContext context) { + if (hasDocValues()) { + return new DocValuesFieldExistsQuery(name()); + } else { + return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); + } + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new QueryShardException(context, "Geo fields do not support exact searching, use dedicated geo queries instead: [" + + name() + "]"); + } + @Override public GeoShapeFieldType clone() { return new GeoShapeFieldType(this); @@ -128,7 +167,18 @@ public void parse(ParseContext context) throws IOException { private void indexShape(ParseContext context, Object luceneShape) { if (luceneShape instanceof Geometry) { - ((Geometry) luceneShape).visit(new LuceneGeometryIndexer(context)); + Geometry geometry = (Geometry) luceneShape; + geometry.visit(new LuceneGeometryIndexer(context)); + if (fieldType().hasDocValues()) { + String name = fieldType().name(); + BinaryGeoShapeDocValuesField docValuesField = (BinaryGeoShapeDocValuesField) context.doc().getByKey(name); + if (docValuesField == null) { + docValuesField = new BinaryGeoShapeDocValuesField(name, geometry); + context.doc().addWithKey(name, docValuesField); + } else { + docValuesField.add(geometry); + } + } } else { throw new IllegalArgumentException("invalid shape type found [" + luceneShape.getClass() + "] while indexing shape"); } @@ -225,4 +275,37 @@ private void indexFields(ParseContext context, Field[] fields) { context.doc().add(f); } } + + static class BinaryGeoShapeDocValuesField extends CustomDocValuesField { + + private List geometries; + + BinaryGeoShapeDocValuesField(String name, Geometry geometry) { + super(name); + this.geometries = new ArrayList<>(1); + add(geometry); + } + + void add(Geometry geometry) { + geometries.add(geometry); + } + + @Override + public BytesRef binaryValue() { + try { + final Geometry geometry; + if (geometries.size() > 1) { + geometry = new GeometryCollection(geometries); + } else { + geometry = geometries.get(0); + } + final GeometryTreeWriter writer = new GeometryTreeWriter(geometry); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + return output.bytes().toBytesRef(); + } catch (IOException e) { + throw new ElasticsearchException("failed to encode shape", e); + } + } + } } diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 7bca4ac920683..a693d4c55ad9f 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -43,7 +43,7 @@ import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.NumericDoubleValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingNumericDoubleValues; @@ -354,7 +354,7 @@ public boolean needsScores() { @Override protected NumericDoubleValues distance(LeafReaderContext context) { - final MultiGeoPointValues geoPointValues = fieldData.load(context).getGeoPointValues(); + final MultiGeoValues geoPointValues = fieldData.load(context).getGeoValues(); return FieldData.replaceMissing(mode.select(new SortingNumericDoubleValues() { @Override public boolean advanceExact(int docId) throws IOException { @@ -362,7 +362,7 @@ public boolean advanceExact(int docId) throws IOException { int n = geoPointValues.docValueCount(); resize(n); for (int i = 0; i < n; i++) { - GeoPoint other = geoPointValues.nextValue(); + MultiGeoValues.GeoValue other = geoPointValues.nextValue(); double distance = distFunction.calculate( origin.lat(), origin.lon(), other.lat(), other.lon(), DistanceUnit.METERS); values[i] = Math.max(0.0d, distance - offset); @@ -380,11 +380,11 @@ public boolean advanceExact(int docId) throws IOException { protected String getDistanceString(LeafReaderContext ctx, int docId) throws IOException { StringBuilder values = new StringBuilder(mode.name()); values.append(" of: ["); - final MultiGeoPointValues geoPointValues = fieldData.load(ctx).getGeoPointValues(); + final MultiGeoValues geoPointValues = fieldData.load(ctx).getGeoValues(); if (geoPointValues.advanceExact(docId)) { final int num = geoPointValues.docValueCount(); for (int i = 0; i < num; i++) { - GeoPoint value = geoPointValues.nextValue(); + MultiGeoValues.GeoValue value = geoPointValues.nextValue(); values.append("Math.max(arcDistance("); values.append(value).append("(=doc value),"); values.append(origin).append("(=origin)) - ").append(offset).append("(=offset), 0)"); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java index 0cc7734ad7685..b07c79f3544bf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java @@ -21,7 +21,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.aggregations.support.ValuesSource; @@ -29,7 +29,7 @@ import java.io.IOException; /** - * Wrapper class to help convert {@link MultiGeoPointValues} + * Wrapper class to help convert {@link MultiGeoValues} * to numeric long values for bucketing. */ class CellIdSource extends ValuesSource.Numeric { @@ -55,7 +55,7 @@ public boolean isFloatingPoint() { @Override public SortedNumericDocValues longValues(LeafReaderContext ctx) { - return new CellValues(valuesSource.geoPointValues(ctx), precision, encoder); + return new CellValues(valuesSource.geoValues(ctx), precision, encoder); } @Override @@ -78,11 +78,11 @@ public interface GeoPointLongEncoder { } private static class CellValues extends AbstractSortingNumericDocValues { - private MultiGeoPointValues geoValues; + private MultiGeoValues geoValues; private int precision; private GeoPointLongEncoder encoder; - protected CellValues(MultiGeoPointValues geoValues, int precision, GeoPointLongEncoder encoder) { + protected CellValues(MultiGeoValues geoValues, int precision, GeoPointLongEncoder encoder) { this.geoValues = geoValues; this.precision = precision; this.encoder = encoder; @@ -93,8 +93,8 @@ public boolean advanceExact(int docId) throws IOException { if (geoValues.advanceExact(docId)) { resize(geoValues.docValueCount()); for (int i = 0; i < docValueCount(); ++i) { - org.elasticsearch.common.geo.GeoPoint target = geoValues.nextValue(); - values[i] = encoder.encode(target.getLon(), target.getLat(), precision); + MultiGeoValues.GeoValue target = geoValues.nextValue(); + values[i] = encoder.encode(target.lon(), target.lat(), precision); } sort(); return true; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java index b99ae657aaee8..979baac80cf90 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceRangeAggregatorFactory.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.aggregations.Aggregator; @@ -108,7 +108,7 @@ public SortedNumericDocValues longValues(LeafReaderContext ctx) { @Override public SortedNumericDoubleValues doubleValues(LeafReaderContext ctx) { - final MultiGeoPointValues geoValues = source.geoPointValues(ctx); + final MultiGeoValues geoValues = source.geoValues(ctx); return GeoUtils.distanceValues(distanceType, units, geoValues, origin); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java index e6d591482be2b..36642f27a4fcf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java @@ -21,11 +21,10 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.DoubleArray; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.LeafBucketCollector; @@ -81,7 +80,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - final MultiGeoPointValues values = valuesSource.geoPointValues(ctx); + final MultiGeoValues values = valuesSource.geoValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -105,7 +104,8 @@ public void collect(int doc, long bucket) throws IOException { final int valuesCount = values.docValueCount(); for (int i = 0; i < valuesCount; ++i) { - GeoPoint value = values.nextValue(); + MultiGeoValues.GeoValue value = values.nextValue(); + double top = tops.get(bucket); if (value.lat() > top) { top = value.lat(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java index 414679f6e2e42..975356c7d6fe3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.DoubleArray; import org.elasticsearch.common.util.LongArray; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.LeafBucketCollector; @@ -67,7 +67,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol return LeafBucketCollector.NO_OP_COLLECTOR; } final BigArrays bigArrays = context.bigArrays(); - final MultiGeoPointValues values = valuesSource.geoPointValues(ctx); + final MultiGeoValues values = valuesSource.geoValues(ctx); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -90,14 +90,14 @@ public void collect(int doc, long bucket) throws IOException { // update the sum for (int i = 0; i < valueCount; ++i) { - GeoPoint value = values.nextValue(); + MultiGeoValues.GeoValue value = values.nextValue(); //latitude - double correctedLat = value.getLat() - compensationLat; + double correctedLat = value.lat() - compensationLat; double newSumLat = sumLat + correctedLat; compensationLat = (newSumLat - sumLat) - correctedLat; sumLat = newSumLat; //longitude - double correctedLon = value.getLon() - compensationLon; + double correctedLon = value.lon() - compensationLon; double newSumLon = sumLon + correctedLon; compensationLon = (newSumLon - sumLon) - correctedLon; sumLon = newSumLon; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index d7b56af2439e0..1dcee23906bb7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -23,10 +23,9 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -356,7 +355,8 @@ static LongUnaryOperator getGlobalMapping(SortedSetDocValues values, SortedSetDo } } - public static ValuesSource.GeoPoint replaceMissing(final ValuesSource.GeoPoint valuesSource, final GeoPoint missing) { + public static ValuesSource.GeoPoint replaceMissing(final ValuesSource.GeoPoint valuesSource, + final MultiGeoValues.GeoPointValue missing) { return new ValuesSource.GeoPoint() { @Override @@ -365,15 +365,15 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc } @Override - public MultiGeoPointValues geoPointValues(LeafReaderContext context) { - final MultiGeoPointValues values = valuesSource.geoPointValues(context); + public MultiGeoValues geoValues(LeafReaderContext context) { + final MultiGeoValues values = valuesSource.geoValues(context); return replaceMissing(values, missing); } }; } - static MultiGeoPointValues replaceMissing(final MultiGeoPointValues values, final GeoPoint missing) { - return new MultiGeoPointValues() { + static MultiGeoValues replaceMissing(final MultiGeoValues values, final MultiGeoValues.GeoValue missing) { + return new MultiGeoValues() { private int count; @@ -395,7 +395,7 @@ public int docValueCount() { } @Override - public GeoPoint nextValue() throws IOException { + public GeoValue nextValue() throws IOException { if (count > 0) { return values.nextValue(); } else { @@ -404,4 +404,21 @@ public GeoPoint nextValue() throws IOException { } }; } + + public static ValuesSource.GeoShape replaceMissing(final ValuesSource.GeoShape valuesSource, + final MultiGeoValues.GeoShapeValue missing) { + return new ValuesSource.GeoShape() { + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return replaceMissing(valuesSource.bytesValues(context), new BytesRef(missing.toString())); + } + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + final MultiGeoValues values = valuesSource.geoValues(context); + return replaceMissing(values, missing); + } + }; + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index 7fd38288a821b..d8436831dd3c0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -35,9 +35,10 @@ import org.elasticsearch.index.fielddata.DocValueBits; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.SortingBinaryDocValues; @@ -485,13 +486,17 @@ public boolean advanceExact(int doc) throws IOException { } } - public abstract static class GeoPoint extends ValuesSource { + public interface Geo { + MultiGeoValues geoValues(LeafReaderContext context); + } + + public abstract static class GeoPoint extends ValuesSource implements Geo { public static final GeoPoint EMPTY = new GeoPoint() { @Override - public MultiGeoPointValues geoPointValues(LeafReaderContext context) { - return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoPoints(); + public MultiGeoValues geoValues(LeafReaderContext context) { + return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoValues(); } @Override @@ -503,12 +508,10 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc @Override public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { - final MultiGeoPointValues geoPoints = geoPointValues(context); + final MultiGeoValues geoPoints = geoValues(context); return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoPoints); } - public abstract MultiGeoPointValues geoPointValues(LeafReaderContext context); - public static class Fielddata extends GeoPoint { protected final IndexGeoPointFieldData indexFieldData; @@ -522,8 +525,49 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) { return indexFieldData.load(context).getBytesValues(); } - public org.elasticsearch.index.fielddata.MultiGeoPointValues geoPointValues(LeafReaderContext context) { - return indexFieldData.load(context).getGeoPointValues(); + public MultiGeoValues geoValues(LeafReaderContext context) { + return indexFieldData.load(context).getGeoValues(); + } + } + } + + public abstract static class GeoShape extends ValuesSource implements Geo { + + public static final GeoShape EMPTY = new GeoShape() { + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoValues(); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); + } + + }; + + @Override + public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { + final MultiGeoValues geoShapes = geoValues(context); + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoShapes); + } + + public static class Fielddata extends GeoShape { + + protected final IndexGeoShapeFieldData indexFieldData; + + public Fielddata(IndexGeoShapeFieldData indexFieldData) { + this.indexFieldData = indexFieldData; + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) { + return indexFieldData.load(context).getBytesValues(); + } + + public MultiGeoValues geoValues(LeafReaderContext context) { + return indexFieldData.load(context).getGeoValues(); } } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index 919d1b752e22c..89ceef8774591 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -24,8 +24,10 @@ import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; @@ -247,6 +249,8 @@ public VS toValuesSource(QueryShardContext context, Function newComparator(String fieldname, int numHits, int sortP return new FieldComparator.DoubleComparator(numHits, null, null) { @Override protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException { - final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues(); + final MultiGeoValues geoPointValues = geoIndexFieldData.load(context).getGeoValues(); final SortedNumericDoubleValues distanceValues = GeoUtils.distanceValues(geoDistance, unit, geoPointValues, localPoints); final NumericDoubleValues selectedValues; diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java index a46fd68a291ab..71b542952e4fe 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractGeoFieldDataTestCase.java @@ -71,20 +71,20 @@ public void testSortMultiValuesFields() { assumeFalse("Only test on non geo_point fields", getFieldDataType().equals("geo_point")); } - protected void assertValues(MultiGeoPointValues values, int docId) throws IOException { + protected void assertValues(MultiGeoValues values, int docId) throws IOException { assertValues(values, docId, false); } - protected void assertMissing(MultiGeoPointValues values, int docId) throws IOException { + protected void assertMissing(MultiGeoValues values, int docId) throws IOException { assertValues(values, docId, true); } - private void assertValues(MultiGeoPointValues values, int docId, boolean missing) throws IOException { + private void assertValues(MultiGeoValues values, int docId, boolean missing) throws IOException { assertEquals(missing == false, values.advanceExact(docId)); if (missing == false) { final int docCount = values.docValueCount(); for (int i = 0; i < docCount; ++i) { - final GeoPoint point = values.nextValue(); + final MultiGeoValues.GeoValue point = values.nextValue(); assertThat(point.lat(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LAT), lessThanOrEqualTo(GeoUtils.MAX_LAT))); assertThat(point.lon(), allOf(greaterThanOrEqualTo(GeoUtils.MIN_LON), lessThanOrEqualTo(GeoUtils.MAX_LON))); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java index a2d2474886381..d1d9d7b2c0bbf 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/GeoFieldDataTests.java @@ -160,7 +160,7 @@ public void testSingleValueAllSet() throws Exception { AtomicFieldData fieldData = indexFieldData.load(readerContext); assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed())); - MultiGeoPointValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoPointValues(); + MultiGeoValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoValues(); assertValues(fieldValues, 0); assertValues(fieldValues, 1); assertValues(fieldValues, 2); @@ -176,7 +176,7 @@ public void testSingleValueWithMissing() throws Exception { AtomicFieldData fieldData = indexFieldData.load(readerContext); assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed())); - MultiGeoPointValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoPointValues(); + MultiGeoValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoValues(); assertValues(fieldValues, 0); assertMissing(fieldValues, 1); assertValues(fieldValues, 2); @@ -192,7 +192,7 @@ public void testMultiValueAllSet() throws Exception { AtomicFieldData fieldData = indexFieldData.load(readerContext); assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed())); - MultiGeoPointValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoPointValues(); + MultiGeoValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoValues(); assertValues(fieldValues, 0); assertValues(fieldValues, 1); assertValues(fieldValues, 2); @@ -208,7 +208,7 @@ public void testMultiValueWithMissing() throws Exception { AtomicFieldData fieldData = indexFieldData.load(readerContext); assertThat(fieldData.ramBytesUsed(), greaterThanOrEqualTo(minRamBytesUsed())); - MultiGeoPointValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoPointValues(); + MultiGeoValues fieldValues = ((AbstractAtomicGeoPointFieldData)fieldData).getGeoValues(); assertValues(fieldValues, 0); assertMissing(fieldValues, 1); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java index 6c199fefbe36c..720b775b0ec42 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java @@ -29,14 +29,14 @@ public class ScriptDocValuesGeoPointsTests extends ESTestCase { - private static MultiGeoPointValues wrap(GeoPoint[][] points) { - return new MultiGeoPointValues() { + private static MultiGeoValues wrap(GeoPoint[][] points) { + return new MultiGeoValues() { GeoPoint[] current; int i; @Override - public GeoPoint nextValue() { - return current[i++]; + public GeoValue nextValue() { + return new GeoPointValue(current[i++]); } @Override @@ -72,7 +72,7 @@ public void testGeoGetLatLon() throws IOException { final double lon2 = randomLon(); GeoPoint[][] points = {{new GeoPoint(lat1, lon1), new GeoPoint(lat2, lon2)}}; - final MultiGeoPointValues values = wrap(points); + final MultiGeoValues values = wrap(points); final ScriptDocValues.GeoPoints script = new ScriptDocValues.GeoPoints(values); script.setNextDocId(1); @@ -90,7 +90,7 @@ public void testGeoDistance() throws IOException { final double lat = randomLat(); final double lon = randomLon(); GeoPoint[][] points = {{new GeoPoint(lat, lon)}}; - final MultiGeoPointValues values = wrap(points); + final MultiGeoValues values = wrap(points); final ScriptDocValues.GeoPoints script = new ScriptDocValues.GeoPoints(values); script.setNextDocId(0); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index fb18cd9903235..f0140c079a9a5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -29,7 +29,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.test.ESTestCase; @@ -346,20 +346,20 @@ public int docValueCount() { public void testMissingGeoPoints() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); - final GeoPoint[][] values = new GeoPoint[numDocs][]; + final MultiGeoValues.GeoPointValue[][] values = new MultiGeoValues.GeoPointValue[numDocs][]; for (int i = 0; i < numDocs; ++i) { - values[i] = new GeoPoint[random().nextInt(4)]; + values[i] = new MultiGeoValues.GeoPointValue[random().nextInt(4)]; for (int j = 0; j < values[i].length; ++j) { - values[i][j] = new GeoPoint(randomDouble() * 90, randomDouble() * 180); + values[i][j] = new MultiGeoValues.GeoPointValue(new GeoPoint(randomDouble() * 90, randomDouble() * 180)); } } - MultiGeoPointValues asGeoValues = new MultiGeoPointValues() { + MultiGeoValues asGeoValues = new MultiGeoValues() { int doc = -1; int i; @Override - public GeoPoint nextValue() { + public GeoValue nextValue() { return values[doc][i++]; } @@ -375,8 +375,9 @@ public int docValueCount() { return values[doc].length; } }; - final GeoPoint missing = new GeoPoint(randomDouble() * 90, randomDouble() * 180); - MultiGeoPointValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); + final MultiGeoValues.GeoPointValue missing = new MultiGeoValues.GeoPointValue( + new GeoPoint(randomDouble() * 90, randomDouble() * 180)); + MultiGeoValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); for (int i = 0; i < numDocs; ++i) { assertTrue(withMissingReplaced.advanceExact(i)); if (values[i].length > 0) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java index d2f73aab3aaa3..ad4672b0b49cd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java @@ -37,6 +37,7 @@ public void testValidOrdinals() { assertThat(ValuesSourceType.NUMERIC.ordinal(), equalTo(1)); assertThat(ValuesSourceType.BYTES.ordinal(), equalTo(2)); assertThat(ValuesSourceType.GEOPOINT.ordinal(), equalTo(3)); + assertThat(ValuesSourceType.GEOSHAPE.ordinal(), equalTo(4)); } @Override @@ -45,6 +46,7 @@ public void testFromString() { assertThat(ValuesSourceType.fromString("numeric"), equalTo(ValuesSourceType.NUMERIC)); assertThat(ValuesSourceType.fromString("bytes"), equalTo(ValuesSourceType.BYTES)); assertThat(ValuesSourceType.fromString("geopoint"), equalTo(ValuesSourceType.GEOPOINT)); + assertThat(ValuesSourceType.fromString("geoshape"), equalTo(ValuesSourceType.GEOSHAPE)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ValuesSourceType.fromString("does_not_exist")); assertThat(e.getMessage(), equalTo("No enum constant org.elasticsearch.search.aggregations.support.ValuesSourceType.DOES_NOT_EXIST")); @@ -57,6 +59,7 @@ public void testReadFrom() throws IOException { assertReadFromStream(1, ValuesSourceType.NUMERIC); assertReadFromStream(2, ValuesSourceType.BYTES); assertReadFromStream(3, ValuesSourceType.GEOPOINT); + assertReadFromStream(4, ValuesSourceType.GEOSHAPE); } @Override @@ -65,5 +68,6 @@ public void testWriteTo() throws IOException { assertWriteToStream(ValuesSourceType.NUMERIC, 1); assertWriteToStream(ValuesSourceType.BYTES, 2); assertWriteToStream(ValuesSourceType.GEOPOINT, 3); + assertWriteToStream(ValuesSourceType.GEOSHAPE, 4); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java index 65b59a6f2ce89..adfc573d83ed6 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java @@ -404,6 +404,10 @@ private static EsField createField(String fieldName, String typeName, Map buildIndices(String[] indexNames, String javaRegex, foundIndices.sort(Comparator.comparing(EsIndex::name)); return foundIndices; } -} \ No newline at end of file +} From 0bb67270aeb5b5bb115bd4797a0797a363e9d985 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 29 Jul 2019 14:14:31 -0700 Subject: [PATCH 12/62] fix lat/lon encoding from Geometries to the trees --- .../org/elasticsearch/common/geo/Extent.java | 2 +- .../common/geo/GeometryTreeWriter.java | 31 ++++-- .../common/geo/GeometryTreeTests.java | 104 ++++++++++++------ 3 files changed, 90 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index 5a3e7bf108c96..95018a1188971 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -29,7 +29,7 @@ * Object representing the extent of a geometry object within a * {@link GeometryTreeWriter} and {@link EdgeTreeWriter}; */ -public final class Extent implements Writeable { +public class Extent implements Writeable { public final int minX; public final int minY; public final int maxX; diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 23447b0db9ede..e939099b3af83 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.geo.geometry.Circle; @@ -104,7 +105,7 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - addWriter(new EdgeTreeWriter(asIntArray(line.getLons()), asIntArray(line.getLats()), false)); + addWriter(new EdgeTreeWriter(asLonEncodedArray(line.getLons()), asLatEncodedArray(line.getLats()), false)); return null; } @@ -114,8 +115,8 @@ public Void visit(MultiLine multiLine) { List x = new ArrayList<>(size); List y = new ArrayList<>(size); for (Line line : multiLine) { - x.add(asIntArray(line.getLons())); - y.add(asIntArray(line.getLats())); + x.add(asLonEncodedArray(line.getLons())); + y.add(asLatEncodedArray(line.getLats())); } addWriter(new EdgeTreeWriter(x, y, false)); return null; @@ -125,7 +126,7 @@ public Void visit(MultiLine multiLine) { public Void visit(Polygon polygon) { // TODO (support holes) LinearRing outerShell = polygon.getPolygon(); - addWriter(new EdgeTreeWriter(asIntArray(outerShell.getLons()), asIntArray(outerShell.getLats()), true)); + addWriter(new EdgeTreeWriter(asLonEncodedArray(outerShell.getLons()), asLatEncodedArray(outerShell.getLats()), true)); return null; } @@ -139,10 +140,12 @@ public Void visit(MultiPolygon multiPolygon) { @Override public Void visit(Rectangle r) { - int[] lats = new int[] { (int) r.getMinLat(), (int) r.getMinLat(), (int) r.getMaxLat(), (int) r.getMaxLat(), - (int) r.getMinLat()}; - int[] lons = new int[] { (int) r.getMinLon(), (int) r.getMaxLon(), (int) r.getMaxLon(), (int) r.getMinLon(), - (int) r.getMinLon()}; + int encodedMinLat = GeoEncodingUtils.encodeLatitude(r.getMinLat()); + int encodedMaxLat = GeoEncodingUtils.encodeLatitude(r.getMaxLat()); + int encodedMinLon = GeoEncodingUtils.encodeLongitude(r.getMinLon()); + int encodedMaxLon = GeoEncodingUtils.encodeLongitude(r.getMaxLon()); + int[] lats = new int[] { encodedMinLat, encodedMinLat, encodedMaxLat, encodedMaxLat, encodedMinLat }; + int[] lons = new int[] { encodedMinLon, encodedMaxLon, encodedMaxLon, encodedMinLon, encodedMinLon }; addWriter(new EdgeTreeWriter(lons, lats, true)); return null; } @@ -171,10 +174,18 @@ public Void visit(Circle circle) { throw new IllegalArgumentException("invalid shape type found [Circle]"); } - private int[] asIntArray(double[] doub) { + private int[] asLonEncodedArray(double[] doub) { int[] intArr = new int[doub.length]; for (int i = 0; i < intArr.length; i++) { - intArr[i] = (int) doub[i]; + intArr[i] = GeoEncodingUtils.encodeLongitude(doub[i]); + } + return intArr; + } + + private int[] asLatEncodedArray(double[] doub) { + int[] intArr = new int[doub.length]; + for (int i = 0; i < intArr.length; i++) { + intArr[i] = GeoEncodingUtils.encodeLatitude(doub[i]); } return intArr; } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index e61abdf75de82..4f8bfbef4e634 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -18,12 +18,15 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.geometry.Geometry; import org.elasticsearch.geo.geometry.Line; import org.elasticsearch.geo.geometry.LinearRing; import org.elasticsearch.geo.geometry.MultiPoint; import org.elasticsearch.geo.geometry.Point; import org.elasticsearch.geo.geometry.Polygon; +import org.elasticsearch.geo.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -35,58 +38,87 @@ public class GeometryTreeTests extends ESTestCase { + private static class GeoExtent extends Extent { + + GeoExtent(int minX, int minY, int maxX, int maxY) { + super(GeoEncodingUtils.encodeLongitude(minX), + GeoEncodingUtils.encodeLatitude(minY), + GeoEncodingUtils.encodeLongitude(maxX), + GeoEncodingUtils.encodeLatitude(maxY)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || o instanceof Extent == false) return false; + Extent extent = (Extent) o; + return minX == extent.minX && + minY == extent.minY && + maxX == extent.maxX && + maxY == extent.maxY; + } + + @Override + public int hashCode() { + return super.hashCode(); + } + + } + public void testRectangleShape() throws IOException { for (int i = 0; i < 1000; i++) { - int minX = randomIntBetween(-180, 170); - int maxX = randomIntBetween(minX + 10, 180); - int minY = randomIntBetween(-90, 80); - int maxY = randomIntBetween(minY + 10, 90); + int minX = randomIntBetween(-80, 70); + int maxX = randomIntBetween(minX + 10, 80); + int minY = randomIntBetween(-80, 70); + int maxY = randomIntBetween(minY + 10, 80); double[] x = new double[]{minX, maxX, maxX, minX, minX}; double[] y = new double[]{minY, minY, maxY, maxY, minY}; - GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(y, x), Collections.emptyList())); + Geometry rectangle = randomBoolean() ? + new Polygon(new LinearRing(y, x), Collections.emptyList()) : new Rectangle(minY, maxY, minX, maxX); + GeometryTreeWriter writer = new GeometryTreeWriter(rectangle); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertThat(reader.getExtent(), equalTo(new Extent(minX, minY, maxX, maxY))); + assertThat(new GeoExtent(minX, minY, maxX, maxY), equalTo(reader.getExtent())); // box-query touches bottom-left corner - assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY))); + assertTrue(reader.intersects(new GeoExtent(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), minX, minY))); // box-query touches bottom-right corner - assertTrue(reader.intersects(new Extent(maxX, minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), minY))); + assertTrue(reader.intersects(new GeoExtent(maxX, minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), minY))); // box-query touches top-right corner - assertTrue(reader.intersects(new Extent(maxX, maxY, maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180)))); + assertTrue(reader.intersects(new GeoExtent(maxX, maxY, maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); // box-query touches top-left corner - assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), maxY, minX, maxY + randomIntBetween(1, 180)))); + assertTrue(reader.intersects(new GeoExtent(minX - randomIntBetween(1, 10), maxY, minX, maxY + randomIntBetween(1, 10)))); // box-query fully-enclosed inside rectangle - assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + assertTrue(reader.intersects(new GeoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, (3 * maxY + minY) / 4))); // box-query fully-contains poly - assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), - maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180)))); + assertTrue(reader.intersects(new GeoExtent(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), + maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); // box-query half-in-half-out-right - assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), + assertTrue(reader.intersects(new GeoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), (3 * maxY + minY) / 4))); // box-query half-in-half-out-left - assertTrue(reader.intersects(new Extent(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + assertTrue(reader.intersects(new GeoExtent(minX - randomIntBetween(1, 10), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, (3 * maxY + minY) / 4))); // box-query half-in-half-out-top - assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), - maxY + randomIntBetween(1, 1000)))); + assertTrue(reader.intersects(new GeoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), + maxY + randomIntBetween(1, 10)))); // box-query half-in-half-out-bottom - assertTrue(reader.intersects(new Extent((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4))); + assertTrue(reader.intersects(new GeoExtent((3 * minX + maxX) / 4, minY - randomIntBetween(1, 10), + maxX + randomIntBetween(1, 10), (3 * maxY + minY) / 4))); // box-query outside to the right - assertFalse(reader.intersects(new Extent(maxX + randomIntBetween(1, 1000), minY, maxX + randomIntBetween(1001, 2000), maxY))); + assertFalse(reader.intersects(new GeoExtent(maxX + randomIntBetween(1, 4), minY, maxX + randomIntBetween(5, 10), maxY))); // box-query outside to the left - assertFalse(reader.intersects(new Extent(maxX - randomIntBetween(1001, 2000), minY, minX - randomIntBetween(1, 1000), maxY))); + assertFalse(reader.intersects(new GeoExtent(maxX - randomIntBetween(5, 10), minY, minX - randomIntBetween(1, 4), maxY))); // box-query outside to the top - assertFalse(reader.intersects(new Extent(minX, maxY + randomIntBetween(1, 1000), maxX, maxY + randomIntBetween(1001, 2000)))); + assertFalse(reader.intersects(new GeoExtent(minX, maxY + randomIntBetween(1, 4), maxX, maxY + randomIntBetween(5, 10)))); // box-query outside to the bottom - assertFalse(reader.intersects(new Extent(minX, minY - randomIntBetween(1001, 2000), maxX, minY - randomIntBetween(1, 1000)))); + assertFalse(reader.intersects(new GeoExtent(minX, minY - randomIntBetween(5, 10), maxX, minY - randomIntBetween(1, 4)))); } } @@ -101,10 +133,10 @@ public void testPacManPolygon() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new Extent(2, -1, 11, 1))); - assertTrue(reader.intersects(new Extent(-12, -12, 12, 12))); - assertTrue(reader.intersects(new Extent(-2, -1, 2, 0))); - assertTrue(reader.intersects(new Extent(-5, -6, 2, -2))); + assertTrue(reader.intersects(new GeoExtent(2, -1, 11, 1))); + assertTrue(reader.intersects(new GeoExtent(-12, -12, 12, 12))); + assertTrue(reader.intersects(new GeoExtent(-2, -1, 2, 0))); + assertTrue(reader.intersects(new GeoExtent(-5, -6, 2, -2))); } public void testPacManClosedLineString() throws Exception { @@ -118,10 +150,10 @@ public void testPacManClosedLineString() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new Extent(2, -1, 11, 1))); - assertTrue(reader.intersects(new Extent(-12, -12, 12, 12))); - assertTrue(reader.intersects(new Extent(-2, -1, 2, 0))); - assertFalse(reader.intersects(new Extent(-5, -6, 2, -2))); + assertTrue(reader.intersects(new GeoExtent(2, -1, 11, 1))); + assertTrue(reader.intersects(new GeoExtent(-12, -12, 12, 12))); + assertTrue(reader.intersects(new GeoExtent(-2, -1, 2, 0))); + assertFalse(reader.intersects(new GeoExtent(-5, -6, 2, -2))); } public void testPacManLineString() throws Exception { @@ -135,10 +167,10 @@ public void testPacManLineString() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new Extent(2, -1, 11, 1))); - assertTrue(reader.intersects(new Extent(-12, -12, 12, 12))); - assertTrue(reader.intersects(new Extent(-2, -1, 2, 0))); - assertFalse(reader.intersects(new Extent(-5, -6, 2, -2))); + assertTrue(reader.intersects(new GeoExtent(2, -1, 11, 1))); + assertTrue(reader.intersects(new GeoExtent(-12, -12, 12, 12))); + assertTrue(reader.intersects(new GeoExtent(-2, -1, 2, 0))); + assertFalse(reader.intersects(new GeoExtent(-5, -6, 2, -2))); } public void testPacManPoints() throws Exception { @@ -169,6 +201,6 @@ public void testPacManPoints() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new Extent(xMin, yMin, xMax, yMax))); + assertTrue(reader.intersects(new GeoExtent(xMin, yMin, xMax, yMax))); } } From e62d1cb10b14414de0ed3d82b5476f1b00f5ebf4 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 1 Aug 2019 18:31:18 -0700 Subject: [PATCH 13/62] fix merge with spatial's shape indexer --- .../mapper/BinaryGeoShapeDocValuesField.java | 63 +++++++++++++++++++ .../index/mapper/GeoShapeFieldMapper.java | 42 ------------- .../index/mapper/GeoShapeIndexer.java | 6 +- .../spatial/index/mapper/ShapeIndexer.java | 12 ++++ 4 files changed, 78 insertions(+), 45 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java new file mode 100644 index 0000000000000..581b5af68ae62 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.geo.GeometryTreeWriter; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.geometry.Geometry; +import org.elasticsearch.geo.geometry.GeometryCollection; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class BinaryGeoShapeDocValuesField extends CustomDocValuesField { + + private List geometries; + + public BinaryGeoShapeDocValuesField(String name, Geometry geometry) { + super(name); + this.geometries = new ArrayList<>(1); + add(geometry); + } + + public void add(Geometry geometry) { + geometries.add(geometry); + } + + @Override + public BytesRef binaryValue() { + try { + final Geometry geometry; + if (geometries.size() > 1) { + geometry = new GeometryCollection(geometries); + } else { + geometry = geometries.get(0); + } + final GeometryTreeWriter writer = new GeometryTreeWriter(geometry); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + return output.bytes().toBytesRef(); + } catch (IOException e) { + throw new ElasticsearchException("failed to encode shape", e); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index d53e9790a1396..9a4ce79e25975 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -23,27 +23,18 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeometryParser; -import org.elasticsearch.common.geo.GeometryTreeWriter; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.geo.geometry.Geometry; -import org.elasticsearch.geo.geometry.GeometryCollection; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.plain.AbstractLatLonShapeDVIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.VectorGeoShapeQueryProcessor; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - /** * FieldMapper for indexing {@link LatLonShape}s. *

@@ -156,37 +147,4 @@ protected String contentType() { return CONTENT_TYPE; } - static class BinaryGeoShapeDocValuesField extends CustomDocValuesField { - - private List geometries; - - BinaryGeoShapeDocValuesField(String name, Geometry geometry) { - super(name); - this.geometries = new ArrayList<>(1); - add(geometry); - } - - void add(Geometry geometry) { - geometries.add(geometry); - } - - @Override - public BytesRef binaryValue() { - try { - final Geometry geometry; - if (geometries.size() > 1) { - geometry = new GeometryCollection(geometries); - } else { - geometry = geometries.get(0); - } - final GeometryTreeWriter writer = new GeometryTreeWriter(geometry); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - return output.bytes().toBytesRef(); - } catch (IOException e) { - throw new ElasticsearchException("failed to encode shape", e); - } - } - - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java index b7b9953940813..4b3476ee53b13 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java @@ -195,10 +195,10 @@ public List indexShape(ParseContext context, Geometry shape) { @Override public void indexDocValueField(ParseContext context, Geometry shape) { - GeoShapeFieldMapper.BinaryGeoShapeDocValuesField docValuesField = - (GeoShapeFieldMapper.BinaryGeoShapeDocValuesField) context.doc().getByKey(name); + BinaryGeoShapeDocValuesField docValuesField = + (BinaryGeoShapeDocValuesField) context.doc().getByKey(name); if (docValuesField == null) { - docValuesField = new GeoShapeFieldMapper.BinaryGeoShapeDocValuesField(name, shape); + docValuesField = new BinaryGeoShapeDocValuesField(name, shape); context.doc().addWithKey(name, docValuesField); } else { docValuesField.add(shape); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java index d76ef95d72a36..fa04bef82c6a4 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java @@ -19,6 +19,7 @@ import org.elasticsearch.geo.geometry.MultiPolygon; import org.elasticsearch.geo.geometry.Point; import org.elasticsearch.index.mapper.AbstractGeometryFieldMapper; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.ParseContext; import java.util.ArrayList; @@ -49,6 +50,17 @@ public List indexShape(ParseContext context, Geometry shape) { return visitor.fields; } + @Override + public void indexDocValueField(ParseContext context, Geometry shape) { + BinaryGeoShapeDocValuesField docValuesField = (BinaryGeoShapeDocValuesField) context.doc().getByKey(name); + if (docValuesField == null) { + docValuesField = new BinaryGeoShapeDocValuesField(name, shape); + context.doc().addWithKey(name, docValuesField); + } else { + docValuesField.add(shape); + } + } + private class LuceneGeometryVisitor implements GeometryVisitor { private List fields = new ArrayList<>(); private String name; From 9778e7687581562d6c42972f92cd237ff6a86bd2 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 5 Aug 2019 13:33:59 -0700 Subject: [PATCH 14/62] Add support for polygons with holes in GeometryTree (#45068) --- .../common/geo/EdgeTreeReader.java | 39 +++++++++++- .../common/geo/EdgeTreeWriter.java | 14 ++--- .../org/elasticsearch/common/geo/Extent.java | 1 + .../common/geo/GeometryTreeReader.java | 2 +- .../common/geo/GeometryTreeWriter.java | 18 ++++-- .../common/geo/PolygonTreeReader.java | 63 +++++++++++++++++++ .../common/geo/PolygonTreeWriter.java | 60 ++++++++++++++++++ .../common/geo/EdgeTreeTests.java | 11 ++-- .../common/geo/GeometryTreeTests.java | 20 ++++++ 9 files changed, 206 insertions(+), 22 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java create mode 100644 server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index 7da1aa66035fb..8baf7aecc7f07 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -83,6 +83,12 @@ boolean containsBottomLeft(Extent extent) throws IOException { return containsBottomLeft(readRoot(input.position()), extent); } + boolean containsFully(Extent extent) throws IOException { + resetInputPosition(); + input.position(input.position() + Extent.WRITEABLE_SIZE_IN_BYTES); // skip extent + return containsFully(readRoot(input.position()), extent); + } + public boolean crosses(Extent extent) throws IOException { resetInputPosition(); @@ -95,7 +101,11 @@ public boolean crosses(Extent extent) throws IOException { } public Edge readRoot(int position) throws IOException { - return readEdge(position); + input.position(position); + if (input.readBoolean()) { + return readEdge(input.position()); + } + return null; } private Edge readEdge(int position) throws IOException { @@ -143,6 +153,33 @@ private boolean containsBottomLeft(Edge root, Extent extent) throws IOException return res; } + /** + * Returns true if every corner in the rectangle query is contained within the tree's edges. + */ + private boolean containsFully(Edge root, Extent extent) throws IOException { + boolean res = false; + if (root.maxY >= extent.minY) { + // is bbox-query contained within linearRing + // cast infinite ray to the right from each corner of the extent + if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX, extent.minY, Integer.MAX_VALUE, extent.minY) + && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX, extent.maxY, Integer.MAX_VALUE, extent.maxY) + && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX, extent.minY, Integer.MAX_VALUE, extent.minY) + && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX, extent.maxY, Integer.MAX_VALUE, extent.maxY) + ) { + res = true; + } + + if (root.rightOffset > 0) { /* has left node */ + res ^= containsBottomLeft(readLeft(root), extent); + } + + if (root.rightOffset > 0 && extent.maxY >= root.minY) { /* no right node if rightOffset == -1 */ + res ^= containsBottomLeft(readRight(root), extent); + } + } + return res; + } + /** * Returns true if the box crosses any edge in this edge subtree * */ diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index 90bdef95d2aac..9000321ec7021 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.geo.geometry.Polygon; import org.elasticsearch.geo.geometry.ShapeType; import java.io.IOException; @@ -39,7 +38,6 @@ public class EdgeTreeWriter extends ShapeTreeWriter { static final int EDGE_SIZE_IN_BYTES = 28; private final Extent extent; - private final boolean hasArea; private final int numShapes; final Edge tree; @@ -47,13 +45,12 @@ public class EdgeTreeWriter extends ShapeTreeWriter { /** * @param x array of the x-coordinate of points. * @param y array of the y-coordinate of points. - * @param hasArea true if edge-tree represents a {@link Polygon} and has a non-zero area, false otherwise. */ - EdgeTreeWriter(int[] x, int[] y, boolean hasArea) { - this(Collections.singletonList(x), Collections.singletonList(y), hasArea); + EdgeTreeWriter(int[] x, int[] y) { + this(Collections.singletonList(x), Collections.singletonList(y)); } - EdgeTreeWriter(List x, List y, boolean hasArea) { + EdgeTreeWriter(List x, List y) { this.numShapes = x.size(); int minX = Integer.MAX_VALUE; int minY = Integer.MAX_VALUE; @@ -84,7 +81,6 @@ public class EdgeTreeWriter extends ShapeTreeWriter { edges.sort(Edge::compareTo); this.extent = new Extent(minX, minY, maxX, maxY); this.tree = createTree(edges, 0, edges.size() - 1); - this.hasArea = hasArea; } @Override @@ -94,13 +90,13 @@ public Extent getExtent() { @Override public ShapeType getShapeType() { - return hasArea ? ShapeType.POLYGON : (numShapes > 1 ? ShapeType.MULTILINESTRING: ShapeType.LINESTRING); + return numShapes > 1 ? ShapeType.MULTILINESTRING: ShapeType.LINESTRING; } @Override public void writeTo(StreamOutput out) throws IOException { extent.writeTo(out); - tree.writeTo(out); + out.writeOptionalWriteable(tree); } private static Edge createTree(List edges, int low, int high) { diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index 95018a1188971..847f2f830ada5 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -30,6 +30,7 @@ * {@link GeometryTreeWriter} and {@link EdgeTreeWriter}; */ public class Extent implements Writeable { + static final int WRITEABLE_SIZE_IN_BYTES = 16; public final int minX; public final int minY; public final int maxX; diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 01c6eaec30beb..5691080a7ce2e 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -76,7 +76,7 @@ public boolean intersects(Extent extent) throws IOException { private static ShapeTreeReader getReader(ShapeType shapeType, ByteBufferStreamInput input) throws IOException { switch (shapeType) { case POLYGON: - return new EdgeTreeReader(input, true); + return new PolygonTreeReader(input); case POINT: case MULTIPOINT: return new Point2DReader(input); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index e939099b3af83..d6cfb470a2183 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; /** @@ -105,7 +106,7 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - addWriter(new EdgeTreeWriter(asLonEncodedArray(line.getLons()), asLatEncodedArray(line.getLats()), false)); + addWriter(new EdgeTreeWriter(asLonEncodedArray(line.getLons()), asLatEncodedArray(line.getLats()))); return null; } @@ -118,15 +119,22 @@ public Void visit(MultiLine multiLine) { x.add(asLonEncodedArray(line.getLons())); y.add(asLatEncodedArray(line.getLats())); } - addWriter(new EdgeTreeWriter(x, y, false)); + addWriter(new EdgeTreeWriter(x, y)); return null; } @Override public Void visit(Polygon polygon) { - // TODO (support holes) LinearRing outerShell = polygon.getPolygon(); - addWriter(new EdgeTreeWriter(asLonEncodedArray(outerShell.getLons()), asLatEncodedArray(outerShell.getLats()), true)); + int numHoles = polygon.getNumberOfHoles(); + List x = new ArrayList<>(numHoles); + List y = new ArrayList<>(numHoles); + for (int i = 0; i < numHoles; i++) { + LinearRing innerRing = polygon.getHole(i); + x.add(asLonEncodedArray(innerRing.getLons())); + y.add(asLatEncodedArray(innerRing.getLats())); + } + addWriter(new PolygonTreeWriter(asLonEncodedArray(outerShell.getLons()), asLatEncodedArray(outerShell.getLats()), x, y)); return null; } @@ -146,7 +154,7 @@ public Void visit(Rectangle r) { int encodedMaxLon = GeoEncodingUtils.encodeLongitude(r.getMaxLon()); int[] lats = new int[] { encodedMinLat, encodedMinLat, encodedMaxLat, encodedMaxLat, encodedMinLat }; int[] lons = new int[] { encodedMinLon, encodedMaxLon, encodedMaxLon, encodedMinLon, encodedMinLon }; - addWriter(new EdgeTreeWriter(lons, lats, true)); + addWriter(new PolygonTreeWriter(lons, lats, Collections.emptyList(), Collections.emptyList())); return null; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java new file mode 100644 index 0000000000000..bfeaabc402e64 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java @@ -0,0 +1,63 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; + +import java.io.IOException; + +/** + * This {@link ShapeTreeReader} understands how to parse polygons + * serialized with the {@link PolygonTreeWriter} + */ +public class PolygonTreeReader implements ShapeTreeReader { + private final EdgeTreeReader outerShell; + private final EdgeTreeReader holes; + + public PolygonTreeReader(ByteBufferStreamInput input) throws IOException { + int outerShellSize = input.readVInt(); + int outerShellPosition = input.position(); + this.outerShell = new EdgeTreeReader(input, true); + input.position(outerShellPosition + outerShellSize); + boolean hasHoles = input.readBoolean(); + if (hasHoles) { + this.holes = new EdgeTreeReader(input, true); + } else { + this.holes = null; + } + } + + public Extent getExtent() throws IOException { + return outerShell.getExtent(); + } + + /** + * Returns true if the rectangle query and the edge tree's shape overlap + */ + @Override + public boolean intersects(Extent extent) throws IOException { + if (holes != null) { + boolean onlyInHole = holes.containsFully(extent); + if (onlyInHole) { + return false; + } + } + return outerShell.intersects(extent); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java new file mode 100644 index 0000000000000..5959f646c940f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java @@ -0,0 +1,60 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.geo.geometry.Polygon; +import org.elasticsearch.geo.geometry.Rectangle; +import org.elasticsearch.geo.geometry.ShapeType; + +import java.io.IOException; +import java.util.List; + +/** + * {@link Polygon} and {@link Rectangle} Shape Tree Writer for use in doc-values + */ +public class PolygonTreeWriter extends ShapeTreeWriter { + + private final EdgeTreeWriter outerShell; + private final EdgeTreeWriter holes; + + public PolygonTreeWriter(int[] x, int[] y, List holesX, List holesY) { + outerShell = new EdgeTreeWriter(x, y); + holes = holesX.isEmpty() ? null : new EdgeTreeWriter(holesX, holesY); + } + + public Extent getExtent() { + return outerShell.getExtent(); + } + + public ShapeType getShapeType() { + return ShapeType.POLYGON; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + // calculate size of outerShell's tree to make it easy to jump to the holes tree quickly when querying + int size = outerShell.tree.size * EdgeTreeWriter.EDGE_SIZE_IN_BYTES + Extent.WRITEABLE_SIZE_IN_BYTES + 1; + out.writeVInt(size); + long startPosition = out.position(); + outerShell.writeTo(out); + assert out.position() == size + startPosition; + out.writeOptionalWriteable(holes); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index ef3a7e1d5f268..8f6620cbebb5f 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -45,7 +45,7 @@ public void testRectangleShape() throws IOException { int maxY = randomIntBetween(minY + 10, 180); int[] x = new int[]{minX, maxX, maxX, minX, minX}; int[] y = new int[]{minY, minY, maxY, maxY, minY}; - EdgeTreeWriter writer = new EdgeTreeWriter(x, y, true); + EdgeTreeWriter writer = new EdgeTreeWriter(x, y); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); @@ -102,7 +102,7 @@ public void testSimplePolygon() throws IOException { int[] x = asIntArray(geo.getPolygon().getLons(), GeoEncodingUtils::encodeLongitude); int[] y = asIntArray(geo.getPolygon().getLats(), GeoEncodingUtils::encodeLatitude); - EdgeTreeWriter writer = new EdgeTreeWriter(x, y, true); + EdgeTreeWriter writer = new EdgeTreeWriter(x, y); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); @@ -134,7 +134,7 @@ public void testPacMan() throws Exception { int yMax = 1;//5; // test cell crossing poly - EdgeTreeWriter writer = new EdgeTreeWriter(px, py, true); + EdgeTreeWriter writer = new EdgeTreeWriter(px, py); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); @@ -144,9 +144,8 @@ public void testPacMan() throws Exception { public void testGetShapeType() { int[] pointCoord = new int[] { 0 }; - assertThat(new EdgeTreeWriter(pointCoord, pointCoord, true).getShapeType(), equalTo(ShapeType.POLYGON)); - assertThat(new EdgeTreeWriter(pointCoord, pointCoord, false).getShapeType(), equalTo(ShapeType.LINESTRING)); - assertThat(new EdgeTreeWriter(List.of(pointCoord, pointCoord), List.of(pointCoord, pointCoord), false).getShapeType(), + assertThat(new EdgeTreeWriter(pointCoord, pointCoord).getShapeType(), equalTo(ShapeType.LINESTRING)); + assertThat(new EdgeTreeWriter(List.of(pointCoord, pointCoord), List.of(pointCoord, pointCoord)).getShapeType(), equalTo(ShapeType.MULTILINESTRING)); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 4f8bfbef4e634..3acdf49415d26 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -139,6 +139,26 @@ public void testPacManPolygon() throws Exception { assertTrue(reader.intersects(new GeoExtent(-5, -6, 2, -2))); } + // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon + public void testPolygonWithHole() throws Exception { + Polygon polyWithHole = new Polygon(new LinearRing(new double[] { -50, -50, 50, 50, -50 }, new double[] { -50, 50, 50, -50, -50 }), + Collections.singletonList(new LinearRing(new double[] { -10, -10, 10, 10, -10 }, new double[] { -10, 10, 10, -10, -10 }))); + + GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + + assertFalse(reader.intersects(new GeoExtent(6, -6, 6, -6))); // in the hole + assertTrue(reader.intersects(new GeoExtent(25, -25, 25, -25))); // on the mainland + assertFalse(reader.intersects(new GeoExtent(51, 51, 52, 52))); // outside of mainland + assertTrue(reader.intersects(new GeoExtent(-60, -60, 60, 60))); // enclosing us completely + assertTrue(reader.intersects(new GeoExtent(49, 49, 51, 51))); // overlapping the mainland + assertTrue(reader.intersects(new GeoExtent(9, 9, 11, 11))); // overlapping the hole + + } + public void testPacManClosedLineString() throws Exception { // pacman double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; From 0e28db9f998161ab2cfc3f6f6993e2e69daac7d4 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 5 Aug 2019 13:35:33 -0700 Subject: [PATCH 15/62] add polygon comb test --- .../common/geo/GeometryTreeTests.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 3acdf49415d26..700fee75c847d 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -156,7 +156,25 @@ public void testPolygonWithHole() throws Exception { assertTrue(reader.intersects(new GeoExtent(-60, -60, 60, 60))); // enclosing us completely assertTrue(reader.intersects(new GeoExtent(49, 49, 51, 51))); // overlapping the mainland assertTrue(reader.intersects(new GeoExtent(9, 9, 11, 11))); // overlapping the hole + } + + public void testCombPolygon() throws Exception { + double[] px = {0, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 0, 0}; + double[] py = {0, 0, 20, 20, 0, 0, 20, 20, 0, 0, 30, 30, 0}; + + double[] hx = {21, 21, 29, 29, 21}; + double[] hy = {1, 20, 20, 1, 1}; + Polygon polyWithHole = new Polygon(new LinearRing(py, px), Collections.singletonList(new LinearRing(hy, hx))); + // test cell crossing poly + GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + assertTrue(reader.intersects(new GeoExtent(5, 10, 5, 10))); + assertFalse(reader.intersects(new GeoExtent(15, 10, 15, 10))); + assertFalse(reader.intersects(new GeoExtent(25, 10, 25, 10))); } public void testPacManClosedLineString() throws Exception { From 977659660f5ead7c064529f12cf16dd7bf62e609 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 6 Aug 2019 07:50:09 -0700 Subject: [PATCH 16/62] move geo-encoding of points to GeometryTree (#45221) moves the encoding of lat/lon from Point2D to GeometryTree --- .../common/geo/GeometryTreeWriter.java | 12 +++++-- .../common/geo/Point2DWriter.java | 35 ++++++++----------- .../common/geo/Point2DTests.java | 16 ++++----- 3 files changed, 31 insertions(+), 32 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index d6cfb470a2183..7ac96865add62 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -160,14 +160,22 @@ public Void visit(Rectangle r) { @Override public Void visit(Point point) { - Point2DWriter writer = new Point2DWriter(point); + int x = GeoEncodingUtils.encodeLongitude(point.getLon()); + int y = GeoEncodingUtils.encodeLatitude(point.getLat()); + Point2DWriter writer = new Point2DWriter(x, y); addWriter(writer); return null; } @Override public Void visit(MultiPoint multiPoint) { - Point2DWriter writer = new Point2DWriter(multiPoint); + int[] x = new int[multiPoint.size()]; + int[] y = new int[x.length]; + for (int i = 0; i < multiPoint.size(); i++) { + x[i] = GeoEncodingUtils.encodeLongitude(multiPoint.get(i).getLon()); + y[i] = GeoEncodingUtils.encodeLatitude(multiPoint.get(i).getLat()); + } + Point2DWriter writer = new Point2DWriter(x, y); addWriter(writer); return null; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java index 7e93dae4a57e7..ba4b0c06d6553 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java @@ -18,10 +18,7 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.geo.geometry.MultiPoint; -import org.elasticsearch.geo.geometry.Point; import org.elasticsearch.geo.geometry.ShapeType; import java.io.IOException; @@ -39,32 +36,28 @@ public class Point2DWriter extends ShapeTreeWriter { // size of a leaf node where searches are done sequentially. static final int LEAF_SIZE = 64; - Point2DWriter(MultiPoint multiPoint) { - int numPoints = multiPoint.size(); + Point2DWriter(int[] x, int[] y) { + assert x.length == y.length; int minX = Integer.MAX_VALUE; int minY = Integer.MAX_VALUE; int maxX = Integer.MIN_VALUE; int maxY = Integer.MIN_VALUE; - coords = new int[numPoints * K]; - int i = 0; - for (Point point : multiPoint) { - int x = GeoEncodingUtils.encodeLongitude(point.getLon()); - int y = GeoEncodingUtils.encodeLatitude(point.getLat()); - minX = Math.min(minX, x); - minY = Math.min(minY, y); - maxX = Math.max(maxX, x); - maxY = Math.max(maxY, y); - coords[2 * i] = x; - coords[2 * i + 1] = y; - i++; + coords = new int[x.length * K]; + for (int i = 0; i < x.length; i++) { + int xi = x[i]; + int yi = y[i]; + minX = Math.min(minX, xi); + minY = Math.min(minY, yi); + maxX = Math.max(maxX, xi); + maxY = Math.max(maxY, yi); + coords[2 * i] = xi; + coords[2 * i + 1] = yi; } - sort(0, numPoints - 1, 0); + sort(0, x.length - 1, 0); this.extent = new Extent(minX, minY, maxX, maxY); } - Point2DWriter(Point point) { - int x = GeoEncodingUtils.encodeLongitude(point.getLon()); - int y = GeoEncodingUtils.encodeLatitude(point.getLat()); + Point2DWriter(int x, int y) { coords = new int[] {x, y}; this.extent = new Extent(x, y, x, y); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java index fb75d5a9f983b..85fae10a9a205 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java @@ -22,15 +22,12 @@ import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geo.geometry.MultiPoint; import org.elasticsearch.geo.geometry.Point; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomShapeGenerator; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; import static org.hamcrest.Matchers.equalTo; @@ -41,7 +38,7 @@ public void testOnePoint() throws IOException { Point point = gen.buildGeometry(); int x = GeoEncodingUtils.encodeLongitude(point.getLon()); int y = GeoEncodingUtils.encodeLatitude(point.getLat()); - Point2DWriter writer = new Point2DWriter(point); + Point2DWriter writer = new Point2DWriter(x, y); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); @@ -63,15 +60,16 @@ public void testPoints() throws IOException { int maxX = randomIntBetween(minX + 10, 180); int minY = randomIntBetween(-90, 80); int maxY = randomIntBetween(minY + 10, 90); - Extent extent = new Extent(GeoEncodingUtils.encodeLongitude(minX), GeoEncodingUtils.encodeLatitude(minY), - GeoEncodingUtils.encodeLongitude(maxX), GeoEncodingUtils.encodeLatitude(maxY)); + Extent extent = new Extent(minX, minY, maxX, maxY); int numPoints = randomIntBetween(2, 1000); - List points = new ArrayList<>(numPoints); + int[] x = new int[numPoints]; + int[] y = new int[numPoints]; for (int j = 0; j < numPoints; j++) { - points.add(new Point(randomDoubleBetween(minY, maxY, true), randomDoubleBetween(minX, maxX, true))); + x[j] = randomIntBetween(minX, maxX); + y[j] = randomIntBetween(minY, maxY); } - Point2DWriter writer = new Point2DWriter(new MultiPoint(points)); + Point2DWriter writer = new Point2DWriter(x, y); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); From b12ab7cc06eb90871f49f25378ed03663fe3ecfa Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 8 Aug 2019 10:12:51 -0700 Subject: [PATCH 17/62] update Extent to store enough info to handle dateline wrapping (#45301) This object represents the bounding box of a shape similar to how InternalGeoBounds operates --- .../common/geo/EdgeTreeReader.java | 43 +++--- .../common/geo/EdgeTreeWriter.java | 47 +++++-- .../org/elasticsearch/common/geo/Extent.java | 125 +++++++++++++++--- .../common/geo/GeometryTreeReader.java | 6 +- .../common/geo/GeometryTreeWriter.java | 30 ++--- .../common/geo/Point2DReader.java | 17 ++- .../common/geo/Point2DWriter.java | 32 +++-- .../index/fielddata/MultiGeoValues.java | 64 +-------- .../support/ValuesSourceConfig.java | 2 +- .../common/geo/EdgeTreeTests.java | 67 +++++++--- .../elasticsearch/common/geo/ExtentTests.java | 87 ++++++++++++ .../common/geo/GeometryTreeTests.java | 102 ++++++-------- .../common/geo/Point2DTests.java | 26 ++-- 13 files changed, 403 insertions(+), 245 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index 8baf7aecc7f07..6d36506ad8ed9 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -60,13 +60,13 @@ public boolean intersects(Extent extent) throws IOException { static Optional checkExtent(StreamInput input, Extent extent) throws IOException { Extent edgeExtent = new Extent(input); - if (edgeExtent.minY > extent.maxY || edgeExtent.maxX < extent.minX - || edgeExtent.maxY < extent.minY || edgeExtent.minX > extent.maxX) { + if (edgeExtent.minY() > extent.maxY() || edgeExtent.maxX() < extent.minX() + || edgeExtent.maxY() < extent.minY() || edgeExtent.minX() > extent.maxX()) { return Optional.of(false); // tree and bbox-query are disjoint } - if (extent.minX <= edgeExtent.minX && extent.minY <= edgeExtent.minY - && extent.maxX >= edgeExtent.maxX && extent.maxY >= edgeExtent.maxY) { + if (extent.minX() <= edgeExtent.minX() && extent.minY() <= edgeExtent.minY() + && extent.maxX() >= edgeExtent.maxX() && extent.maxY() >= edgeExtent.maxY()) { return Optional.of(true); // bbox-query fully contains tree's extent. } return Optional.empty(); @@ -135,10 +135,11 @@ Edge readRight(Edge root) throws IOException { */ private boolean containsBottomLeft(Edge root, Extent extent) throws IOException { boolean res = false; - if (root.maxY >= extent.minY) { + if (root.maxY >= extent.minY()) { // is bbox-query contained within linearRing // cast infinite ray to the right from bottom-left of bbox-query to see if it intersects edge - if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX, extent.minY, Integer.MAX_VALUE, extent.minY)) { + if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX(), extent.minY(), Integer.MAX_VALUE, + extent.minY())) { res = true; } @@ -146,7 +147,7 @@ private boolean containsBottomLeft(Edge root, Extent extent) throws IOException res ^= containsBottomLeft(readLeft(root), extent); } - if (root.rightOffset > 0 && extent.maxY >= root.minY) { /* no right node if rightOffset == -1 */ + if (root.rightOffset > 0 && extent.maxY() >= root.minY) { /* no right node if rightOffset == -1 */ res ^= containsBottomLeft(readRight(root), extent); } } @@ -158,13 +159,17 @@ private boolean containsBottomLeft(Edge root, Extent extent) throws IOException */ private boolean containsFully(Edge root, Extent extent) throws IOException { boolean res = false; - if (root.maxY >= extent.minY) { + if (root.maxY >= extent.minY()) { // is bbox-query contained within linearRing // cast infinite ray to the right from each corner of the extent - if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX, extent.minY, Integer.MAX_VALUE, extent.minY) - && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX, extent.maxY, Integer.MAX_VALUE, extent.maxY) - && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX, extent.minY, Integer.MAX_VALUE, extent.minY) - && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX, extent.maxY, Integer.MAX_VALUE, extent.maxY) + if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX(), extent.minY(), + Integer.MAX_VALUE, extent.minY()) + && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX(), extent.maxY(), + Integer.MAX_VALUE, extent.maxY()) + && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX(), extent.minY(), + Integer.MAX_VALUE, extent.minY()) + && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX(), extent.maxY(), + Integer.MAX_VALUE, extent.maxY()) ) { res = true; } @@ -173,7 +178,7 @@ && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX, res ^= containsBottomLeft(readLeft(root), extent); } - if (root.rightOffset > 0 && extent.maxY >= root.minY) { /* no right node if rightOffset == -1 */ + if (root.rightOffset > 0 && extent.maxY() >= root.minY) { /* no right node if rightOffset == -1 */ res ^= containsBottomLeft(readRight(root), extent); } } @@ -185,17 +190,17 @@ && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX, * */ private boolean crosses(Edge root, Extent extent) throws IOException { // we just have to cross one edge to answer the question, so we descend the tree and return when we do. - if (root.maxY >= extent.minY) { + if (root.maxY >= extent.minY()) { // does rectangle's edges intersect or reside inside polygon's edge if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, - extent.minX, extent.minY, extent.maxX, extent.minY) || + extent.minX(), extent.minY(), extent.maxX(), extent.minY()) || lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, - extent.maxX, extent.minY, extent.maxX, extent.maxY) || + extent.maxX(), extent.minY(), extent.maxX(), extent.maxY()) || lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, - extent.maxX, extent.maxY, extent.minX, extent.maxY) || + extent.maxX(), extent.maxY(), extent.minX(), extent.maxY()) || lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, - extent.minX, extent.maxY, extent.minX, extent.minY)) { + extent.minX(), extent.maxY(), extent.minX(), extent.minY())) { return true; } /* has left node */ @@ -204,7 +209,7 @@ private boolean crosses(Edge root, Extent extent) throws IOException { } /* no right node if rightOffset == -1 */ - if (root.rightOffset > 0 && extent.maxY >= root.minY && crosses(readRight(root), extent)) { + if (root.rightOffset > 0 && extent.maxY() >= root.minY && crosses(readRight(root), extent)) { return true; } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index 9000321ec7021..6da82c4e13eb0 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -52,10 +52,12 @@ public class EdgeTreeWriter extends ShapeTreeWriter { EdgeTreeWriter(List x, List y) { this.numShapes = x.size(); - int minX = Integer.MAX_VALUE; - int minY = Integer.MAX_VALUE; - int maxX = Integer.MIN_VALUE; - int maxY = Integer.MIN_VALUE; + int top = Integer.MIN_VALUE; + int bottom = Integer.MAX_VALUE; + int negLeft = Integer.MAX_VALUE; + int negRight = Integer.MIN_VALUE; + int posLeft = Integer.MAX_VALUE; + int posRight = Integer.MIN_VALUE; List edges = new ArrayList<>(); for (int i = 0; i < y.size(); i++) { for (int j = 1; j < y.get(i).length; j++) { @@ -72,14 +74,41 @@ public class EdgeTreeWriter extends ShapeTreeWriter { edgeMaxY = y1; } edges.add(new Edge(x1, y1, x2, y2, edgeMinY, edgeMaxY)); - minX = Math.min(minX, Math.min(x1, x2)); - minY = Math.min(minY, Math.min(y1, y2)); - maxX = Math.max(maxX, Math.max(x1, x2)); - maxY = Math.max(maxY, Math.max(y1, y2)); + + top = Math.max(top, Math.max(y1, y2)); + bottom = Math.min(bottom, Math.min(y1, y2)); + + // check first + if (x1 >= 0 && x1 < posLeft) { + posLeft = x1; + } + if (x1 >= 0 && x1 > posRight) { + posRight = x1; + } + if (x1 < 0 && x1 < negLeft) { + negLeft = x1; + } + if (x1 < 0 && x1 > negRight) { + negRight = x1; + } + + // check second + if (x2 >= 0 && x2 < posLeft) { + posLeft = x2; + } + if (x2 >= 0 && x2 > posRight) { + posRight = x2; + } + if (x2 < 0 && x2 < negLeft) { + negLeft = x2; + } + if (x2 < 0 && x2 > negRight) { + negRight = x2; + } } } edges.sort(Edge::compareTo); - this.extent = new Extent(minX, minY, maxX, maxY); + this.extent = new Extent(top, bottom, negLeft, negRight, posLeft, posRight); this.tree = createTree(edges, 0, edges.size() - 1); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index 847f2f830ada5..ebf2fabbac349 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -27,32 +27,111 @@ /** * Object representing the extent of a geometry object within a - * {@link GeometryTreeWriter} and {@link EdgeTreeWriter}; + * {@link GeometryTreeWriter} and {@link EdgeTreeWriter}. */ public class Extent implements Writeable { - static final int WRITEABLE_SIZE_IN_BYTES = 16; - public final int minX; - public final int minY; - public final int maxX; - public final int maxY; - - Extent(int minX, int minY, int maxX, int maxY) { - this.minX = minX; - this.minY = minY; - this.maxX = maxX; - this.maxY = maxY; + static final int WRITEABLE_SIZE_IN_BYTES = 24; + + public final int top; + public final int bottom; + public final int negLeft; + public final int negRight; + public final int posLeft; + public final int posRight; + + Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + this.top = top; + this.bottom = bottom; + this.negLeft = negLeft; + this.negRight = negRight; + this.posLeft = posLeft; + this.posRight = posRight; } Extent(StreamInput input) throws IOException { - this(input.readInt(), input.readInt(), input.readInt(), input.readInt()); + this(input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt()); + } + + /** + * calculates the extent of a point, which is the point itself. + * @param x the x-coordinate of the point + * @param y the y-coordinate of the point + * @return the extent of the point + */ + static Extent fromPoint(int x, int y) { + return new Extent(y, y, + x < 0 ? x : Integer.MAX_VALUE, + x < 0 ? x : Integer.MIN_VALUE, + x >= 0 ? x : Integer.MAX_VALUE, + x >= 0 ? x : Integer.MIN_VALUE); + } + + /** + * calculates the extent of two points representing a bounding box's bottom-left + * and top-right points. It is important that these points accurately represent the + * bottom-left and top-right of the extent since there is no validation being done. + * + * @param bottomLeftX the bottom-left x-coordinate + * @param bottomLeftY the bottom-left y-coordinate + * @param topRightX the top-right x-coordinate + * @param topRightY the top-right y-coordinate + * @return the extent of the two points + */ + static Extent fromPoints(int bottomLeftX, int bottomLeftY, int topRightX, int topRightY) { + int negLeft = Integer.MAX_VALUE; + int negRight = Integer.MIN_VALUE; + int posLeft = Integer.MAX_VALUE; + int posRight = Integer.MIN_VALUE; + if (bottomLeftX < 0 && topRightX < 0) { + negLeft = bottomLeftX; + negRight = topRightX; + } else if (bottomLeftX < 0) { + negLeft = negRight = bottomLeftX; + posLeft = posRight = topRightX; + } else { + posLeft = bottomLeftX; + posRight = topRightX; + } + return new Extent(topRightY, bottomLeftY, negLeft, negRight, posLeft, posRight); + } + + /** + * @return the minimum y-coordinate of the extent + */ + public int minY() { + return bottom; } + /** + * @return the maximum y-coordinate of the extent + */ + public int maxY() { + return top; + } + + /** + * @return the absolute minimum x-coordinate of the extent, whether it is positive or negative. + */ + public int minX() { + return Math.min(negLeft, posLeft); + } + + /** + * @return the absolute maximum x-coordinate of the extent, whether it is positive or negative. + */ + public int maxX() { + return Math.max(negRight, posRight); + } + + @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(minX); - out.writeInt(minY); - out.writeInt(maxX); - out.writeInt(maxY); + out.writeInt(top); + out.writeInt(bottom); + out.writeInt(negLeft); + out.writeInt(negRight); + out.writeInt(posLeft); + out.writeInt(posRight); } @Override @@ -60,14 +139,16 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Extent extent = (Extent) o; - return minX == extent.minX && - minY == extent.minY && - maxX == extent.maxX && - maxY == extent.maxY; + return top == extent.top && + bottom == extent.bottom && + negLeft == extent.negLeft && + negRight == extent.negRight && + posLeft == extent.posLeft && + posRight == extent.posRight; } @Override public int hashCode() { - return Objects.hash(minX, minY, maxX, maxY); + return Objects.hash(top, bottom, negLeft, negRight, posLeft, posRight); } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 5691080a7ce2e..d93fd77b8646f 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -42,9 +42,9 @@ public GeometryTreeReader(BytesRef bytesRef) { public Extent getExtent() throws IOException { input.position(0); - boolean hasExtent = input.readBoolean(); - if (hasExtent) { - return new Extent(input); + Extent extent = input.readOptionalWriteable(Extent::new); + if (extent != null) { + return extent; } assert input.readVInt() == 1; ShapeType shapeType = input.readEnum(ShapeType.class); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 7ac96865add62..22e0757b90ff2 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -58,13 +58,11 @@ public void writeTo(StreamOutput out) throws IOException { // only write a geometry extent for the tree if the tree // contains multiple sub-shapes boolean prependExtent = builder.shapeWriters.size() > 1; - out.writeBoolean(prependExtent); + Extent extent = null; if (prependExtent) { - out.writeInt(builder.minLon); - out.writeInt(builder.minLat); - out.writeInt(builder.maxLon); - out.writeInt(builder.maxLat); + extent = new Extent(builder.top, builder.bottom, builder.negLeft, builder.negRight, builder.posLeft, builder.posRight); } + out.writeOptionalWriteable(extent); out.writeVInt(builder.shapeWriters.size()); for (ShapeTreeWriter writer : builder.shapeWriters) { out.writeEnum(writer.getShapeType()); @@ -76,23 +74,25 @@ class GeometryTreeBuilder implements GeometryVisitor { private List shapeWriters; // integers are used to represent int-encoded lat/lon values - int minLat; - int maxLat; - int minLon; - int maxLon; + int top = Integer.MIN_VALUE; + int bottom = Integer.MAX_VALUE; + int negLeft = Integer.MAX_VALUE; + int negRight = Integer.MIN_VALUE; + int posLeft = Integer.MAX_VALUE; + int posRight = Integer.MIN_VALUE; GeometryTreeBuilder() { shapeWriters = new ArrayList<>(); - minLat = minLon = Integer.MAX_VALUE; - maxLat = maxLon = Integer.MIN_VALUE; } private void addWriter(ShapeTreeWriter writer) { Extent extent = writer.getExtent(); - minLon = Math.min(minLon, extent.minX); - minLat = Math.min(minLat, extent.minY); - maxLon = Math.max(maxLon, extent.maxX); - maxLat = Math.max(maxLat, extent.maxY); + top = Math.max(top, extent.top); + bottom = Math.min(bottom, extent.bottom); + negLeft = Math.min(negLeft, extent.negLeft); + negRight = Math.max(negRight, extent.negRight); + posLeft = Math.min(posLeft, extent.posLeft); + posRight = Math.max(posRight, extent.posRight); shapeWriters.add(writer); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java index 207220e108122..9ab106be76092 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java @@ -43,8 +43,9 @@ public Extent getExtent() throws IOException { if (size == 1) { int x = readX(0); int y = readY(0); - return new Extent(x, y, x, y); + return Extent.fromPoint(x, y); } else { + input.position(startPosition); return new Extent(input); } } @@ -64,7 +65,7 @@ public boolean intersects(Extent extent) throws IOException { // TODO serialize to re-usable array instead of serializing in each step int x = readX(i); int y = readY(i); - if (x >= extent.minX && x <= extent.maxX && y >= extent.minY && y <= extent.maxY) { + if (x >= extent.minX() && x <= extent.maxX() && y >= extent.minY() && y <= extent.maxY()) { return true; } } @@ -74,15 +75,15 @@ public boolean intersects(Extent extent) throws IOException { int middle = (right - left) >> 1; int x = readX(middle); int y = readY(middle); - if (x >= extent.minX && x <= extent.maxX && y >= extent.minY && y <= extent.maxY) { + if (x >= extent.minX() && x <= extent.maxX() && y >= extent.minY() && y <= extent.maxY()) { return true; } - if ((axis == 0 && extent.minX <= x) || (axis == 1 && extent.minY <= y)) { + if ((axis == 0 && extent.minX() <= x) || (axis == 1 && extent.minY() <= y)) { stack.push(left); stack.push(middle - 1); stack.push(1 - axis); } - if ((axis == 0 && extent.maxX >= x) || (axis == 1 && extent.maxY >= y)) { + if ((axis == 0 && extent.maxX() >= x) || (axis == 1 && extent.maxY() >= y)) { stack.push(middle + 1); stack.push(right); stack.push(1 - axis); @@ -93,12 +94,14 @@ public boolean intersects(Extent extent) throws IOException { } private int readX(int pointIdx) throws IOException { - input.position(startPosition + 2 * pointIdx * Integer.BYTES); + int extentOffset = size == 1 ? 0 : Extent.WRITEABLE_SIZE_IN_BYTES; + input.position(startPosition + extentOffset + 2 * pointIdx * Integer.BYTES); return input.readInt(); } private int readY(int pointIdx) throws IOException { - input.position(startPosition + (2 * pointIdx + 1) * Integer.BYTES); + int extentOffset = size == 1 ? 0 : Extent.WRITEABLE_SIZE_IN_BYTES; + input.position(startPosition + extentOffset + (2 * pointIdx + 1) * Integer.BYTES); return input.readInt(); } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java index ba4b0c06d6553..837a069dd5f13 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java @@ -38,28 +38,40 @@ public class Point2DWriter extends ShapeTreeWriter { Point2DWriter(int[] x, int[] y) { assert x.length == y.length; - int minX = Integer.MAX_VALUE; - int minY = Integer.MAX_VALUE; - int maxX = Integer.MIN_VALUE; - int maxY = Integer.MIN_VALUE; + int top = Integer.MIN_VALUE; + int bottom = Integer.MAX_VALUE; + int negLeft = Integer.MAX_VALUE; + int negRight = Integer.MIN_VALUE; + int posLeft = Integer.MAX_VALUE; + int posRight = Integer.MIN_VALUE; coords = new int[x.length * K]; for (int i = 0; i < x.length; i++) { int xi = x[i]; int yi = y[i]; - minX = Math.min(minX, xi); - minY = Math.min(minY, yi); - maxX = Math.max(maxX, xi); - maxY = Math.max(maxY, yi); + top = Math.max(top, yi); + bottom = Math.min(bottom, yi); + if (xi >= 0 && xi < posLeft) { + posLeft = xi; + } + if (xi >= 0 && xi > posRight) { + posRight = xi; + } + if (xi < 0 && xi < negLeft) { + negLeft = xi; + } + if (xi < 0 && xi > negRight) { + negRight = xi; + } coords[2 * i] = xi; coords[2 * i + 1] = yi; } sort(0, x.length - 1, 0); - this.extent = new Extent(minX, minY, maxX, maxY); + this.extent = new Extent(top, bottom, negLeft, negRight, posLeft, posRight); } Point2DWriter(int x, int y) { coords = new int[] {x, y}; - this.extent = new Extent(x, y, x, y); + this.extent = Extent.fromPoint(x, y); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 958e79a071ea9..38a24643ea039 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.index.fielddata; -import org.apache.lucene.geo.GeoEncodingUtils; -import org.elasticsearch.common.geo.Extent; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeometryTreeReader; @@ -80,26 +78,6 @@ public GeoPoint geoPoint() { return geoPoint; } - @Override - public double minLat() { - return geoPoint.lat(); - } - - @Override - public double maxLat() { - return geoPoint.lat(); - } - - @Override - public double minLon() { - return geoPoint.lon(); - } - - @Override - public double maxLon() { - return geoPoint.lon(); - } - @Override public double lat() { return geoPoint.lat(); @@ -117,44 +95,14 @@ public String toString() { } public static class GeoShapeValue implements GeoValue { - private final double minLat; - private final double maxLat; - private final double minLon; - private final double maxLon; + private final GeometryTreeReader reader; public GeoShapeValue(GeometryTreeReader reader) throws IOException { - Extent extent = reader.getExtent(); - this.minLat = GeoEncodingUtils.decodeLatitude(extent.minY); - this.maxLat = GeoEncodingUtils.decodeLatitude(extent.maxY); - this.maxLon = GeoEncodingUtils.decodeLongitude(extent.maxX); - this.minLon = GeoEncodingUtils.decodeLongitude(extent.minX); - } - - public GeoShapeValue(double minLat, double minLon, double maxLat, double maxLon) { - this.minLat = minLat; - this.minLon = minLon; - this.maxLat = maxLat; - this.maxLon = maxLon; - } - - @Override - public double minLat() { - return minLat; + this.reader = reader; } - @Override - public double maxLat() { - return maxLat; - } - - @Override - public double minLon() { - return minLon; - } - - @Override - public double maxLon() { - return maxLon; + public GeoShapeValue() { + this.reader = null; } @Override @@ -173,10 +121,6 @@ public double lon() { * retrieve properties used in aggregations. */ public interface GeoValue { - double minLat(); - double maxLat(); - double minLon(); - double maxLon(); double lat(); double lon(); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index 89ceef8774591..0aa2af2271c20 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -282,7 +282,7 @@ public VS toValuesSource(QueryShardContext context, Function= minYBox) { - assertTrue(reader.intersects(new Extent(minXBox, minYBox, maxXBox, maxYBox - 1))); + assertTrue(reader.intersects(Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1))); } if (maxXBox -1 >= minXBox) { - assertTrue(reader.intersects(new Extent(minXBox, minYBox, maxXBox - 1, maxYBox))); + assertTrue(reader.intersects(Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox))); } // does not cross - assertFalse(reader.intersects(new Extent(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10))); + assertFalse(reader.intersects(Extent.fromPoints(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10))); } } @@ -139,7 +162,7 @@ public void testPacMan() throws Exception { writer.writeTo(output); output.close(); EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); - assertTrue(reader.containsBottomLeft(new Extent(xMin, yMin, xMax, yMax))); + assertTrue(reader.containsBottomLeft(Extent.fromPoints(xMin, yMin, xMax, yMax))); } public void testGetShapeType() { diff --git a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java new file mode 100644 index 0000000000000..2c527845240c2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java @@ -0,0 +1,87 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class ExtentTests extends AbstractWireSerializingTestCase { + + public void testFromPoint() { + int x = randomFrom(-1, 0, 1); + int y = randomFrom(-1, 0, 1); + Extent extent = Extent.fromPoint(x, y); + assertThat(extent.minX(), equalTo(x)); + assertThat(extent.maxX(), equalTo(x)); + assertThat(extent.minY(), equalTo(y)); + assertThat(extent.maxY(), equalTo(y)); + } + + public void testFromPoints() { + int bottomLeftX = randomFrom(-10, 0, 10); + int bottomLeftY = randomFrom(-10, 0, 10); + int topRightX = bottomLeftX + randomIntBetween(0, 20); + int topRightY = bottomLeftX + randomIntBetween(0, 20); + Extent extent = Extent.fromPoints(bottomLeftX, bottomLeftY, topRightX, topRightY); + assertThat(extent.minX(), equalTo(bottomLeftX)); + assertThat(extent.maxX(), equalTo(topRightX)); + assertThat(extent.minY(), equalTo(bottomLeftY)); + assertThat(extent.maxY(), equalTo(topRightY)); + assertThat(extent.top, equalTo(topRightY)); + assertThat(extent.bottom, equalTo(bottomLeftY)); + if (bottomLeftX < 0 && topRightX < 0) { + assertThat(extent.negLeft, equalTo(bottomLeftX)); + assertThat(extent.negRight, equalTo(topRightX)); + assertThat(extent.posLeft, equalTo(Integer.MAX_VALUE)); + assertThat(extent.posRight, equalTo(Integer.MIN_VALUE)); + } else if (bottomLeftX < 0) { + assertThat(extent.negLeft, equalTo(bottomLeftX)); + assertThat(extent.negRight, equalTo(bottomLeftX)); + assertThat(extent.posLeft, equalTo(topRightX)); + assertThat(extent.posRight, equalTo(topRightX)); + } else { + assertThat(extent.negLeft, equalTo(Integer.MAX_VALUE)); + assertThat(extent.negRight, equalTo(Integer.MIN_VALUE)); + assertThat(extent.posLeft, equalTo(bottomLeftX)); + assertThat(extent.posRight, equalTo(topRightX)); + } + } + + @Override + protected Extent createTestInstance() { + return new Extent(randomIntBetween(-10, 10), randomIntBetween(-10, 10), randomIntBetween(-10, 10), + randomIntBetween(-10, 10), randomIntBetween(-10, 10), randomIntBetween(-10, 10)); + } + + @Override + protected Writeable.Reader instanceReader() { + return Extent::new; + } + + @Override + protected Object copyInstance(Object instance, Version version) throws IOException { + Extent other = (Extent) instance; + return new Extent(other.top, other.bottom, other.negLeft, other.negRight, other.posLeft, other.posRight); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 700fee75c847d..fee6389ec8544 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -38,31 +38,9 @@ public class GeometryTreeTests extends ESTestCase { - private static class GeoExtent extends Extent { - - GeoExtent(int minX, int minY, int maxX, int maxY) { - super(GeoEncodingUtils.encodeLongitude(minX), - GeoEncodingUtils.encodeLatitude(minY), - GeoEncodingUtils.encodeLongitude(maxX), - GeoEncodingUtils.encodeLatitude(maxY)); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || o instanceof Extent == false) return false; - Extent extent = (Extent) o; - return minX == extent.minX && - minY == extent.minY && - maxX == extent.maxX && - maxY == extent.maxY; - } - - @Override - public int hashCode() { - return super.hashCode(); - } - + static Extent geoExtent(double minX, double minY, double maxX, double maxY) { + return Extent.fromPoints(GeoEncodingUtils.encodeLongitude(minX), GeoEncodingUtils.encodeLatitude(minY), + GeoEncodingUtils.encodeLongitude(maxX), GeoEncodingUtils.encodeLatitude(maxY)); } public void testRectangleShape() throws IOException { @@ -82,43 +60,43 @@ public void testRectangleShape() throws IOException { output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertThat(new GeoExtent(minX, minY, maxX, maxY), equalTo(reader.getExtent())); + assertThat(geoExtent(minX, minY, maxX, maxY), equalTo(reader.getExtent())); // box-query touches bottom-left corner - assertTrue(reader.intersects(new GeoExtent(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), minX, minY))); + assertTrue(reader.intersects(geoExtent(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), minX, minY))); // box-query touches bottom-right corner - assertTrue(reader.intersects(new GeoExtent(maxX, minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), minY))); + assertTrue(reader.intersects(geoExtent(maxX, minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), minY))); // box-query touches top-right corner - assertTrue(reader.intersects(new GeoExtent(maxX, maxY, maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); + assertTrue(reader.intersects(geoExtent(maxX, maxY, maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); // box-query touches top-left corner - assertTrue(reader.intersects(new GeoExtent(minX - randomIntBetween(1, 10), maxY, minX, maxY + randomIntBetween(1, 10)))); + assertTrue(reader.intersects(geoExtent(minX - randomIntBetween(1, 10), maxY, minX, maxY + randomIntBetween(1, 10)))); // box-query fully-enclosed inside rectangle - assertTrue(reader.intersects(new GeoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + assertTrue(reader.intersects(geoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, (3 * maxY + minY) / 4))); // box-query fully-contains poly - assertTrue(reader.intersects(new GeoExtent(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), + assertTrue(reader.intersects(geoExtent(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); // box-query half-in-half-out-right - assertTrue(reader.intersects(new GeoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), + assertTrue(reader.intersects(geoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), (3 * maxY + minY) / 4))); // box-query half-in-half-out-left - assertTrue(reader.intersects(new GeoExtent(minX - randomIntBetween(1, 10), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + assertTrue(reader.intersects(geoExtent(minX - randomIntBetween(1, 10), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, (3 * maxY + minY) / 4))); // box-query half-in-half-out-top - assertTrue(reader.intersects(new GeoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), + assertTrue(reader.intersects(geoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); // box-query half-in-half-out-bottom - assertTrue(reader.intersects(new GeoExtent((3 * minX + maxX) / 4, minY - randomIntBetween(1, 10), + assertTrue(reader.intersects(geoExtent((3 * minX + maxX) / 4, minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), (3 * maxY + minY) / 4))); // box-query outside to the right - assertFalse(reader.intersects(new GeoExtent(maxX + randomIntBetween(1, 4), minY, maxX + randomIntBetween(5, 10), maxY))); + assertFalse(reader.intersects(geoExtent(maxX + randomIntBetween(1, 4), minY, maxX + randomIntBetween(5, 10), maxY))); // box-query outside to the left - assertFalse(reader.intersects(new GeoExtent(maxX - randomIntBetween(5, 10), minY, minX - randomIntBetween(1, 4), maxY))); + assertFalse(reader.intersects(geoExtent(maxX - randomIntBetween(5, 10), minY, minX - randomIntBetween(1, 4), maxY))); // box-query outside to the top - assertFalse(reader.intersects(new GeoExtent(minX, maxY + randomIntBetween(1, 4), maxX, maxY + randomIntBetween(5, 10)))); + assertFalse(reader.intersects(geoExtent(minX, maxY + randomIntBetween(1, 4), maxX, maxY + randomIntBetween(5, 10)))); // box-query outside to the bottom - assertFalse(reader.intersects(new GeoExtent(minX, minY - randomIntBetween(5, 10), maxX, minY - randomIntBetween(1, 4)))); + assertFalse(reader.intersects(geoExtent(minX, minY - randomIntBetween(5, 10), maxX, minY - randomIntBetween(1, 4)))); } } @@ -133,10 +111,10 @@ public void testPacManPolygon() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new GeoExtent(2, -1, 11, 1))); - assertTrue(reader.intersects(new GeoExtent(-12, -12, 12, 12))); - assertTrue(reader.intersects(new GeoExtent(-2, -1, 2, 0))); - assertTrue(reader.intersects(new GeoExtent(-5, -6, 2, -2))); + assertTrue(reader.intersects(geoExtent(2, -1, 11, 1))); + assertTrue(reader.intersects(geoExtent(-12, -12, 12, 12))); + assertTrue(reader.intersects(geoExtent(-2, -1, 2, 0))); + assertTrue(reader.intersects(geoExtent(-5, -6, 2, -2))); } // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon @@ -150,12 +128,12 @@ public void testPolygonWithHole() throws Exception { output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertFalse(reader.intersects(new GeoExtent(6, -6, 6, -6))); // in the hole - assertTrue(reader.intersects(new GeoExtent(25, -25, 25, -25))); // on the mainland - assertFalse(reader.intersects(new GeoExtent(51, 51, 52, 52))); // outside of mainland - assertTrue(reader.intersects(new GeoExtent(-60, -60, 60, 60))); // enclosing us completely - assertTrue(reader.intersects(new GeoExtent(49, 49, 51, 51))); // overlapping the mainland - assertTrue(reader.intersects(new GeoExtent(9, 9, 11, 11))); // overlapping the hole + assertFalse(reader.intersects(geoExtent(6, -6, 6, -6))); // in the hole + assertTrue(reader.intersects(geoExtent(25, -25, 25, -25))); // on the mainland + assertFalse(reader.intersects(geoExtent(51, 51, 52, 52))); // outside of mainland + assertTrue(reader.intersects(geoExtent(-60, -60, 60, 60))); // enclosing us completely + assertTrue(reader.intersects(geoExtent(49, 49, 51, 51))); // overlapping the mainland + assertTrue(reader.intersects(geoExtent(9, 9, 11, 11))); // overlapping the hole } public void testCombPolygon() throws Exception { @@ -172,9 +150,9 @@ public void testCombPolygon() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new GeoExtent(5, 10, 5, 10))); - assertFalse(reader.intersects(new GeoExtent(15, 10, 15, 10))); - assertFalse(reader.intersects(new GeoExtent(25, 10, 25, 10))); + assertTrue(reader.intersects(geoExtent(5, 10, 5, 10))); + assertFalse(reader.intersects(geoExtent(15, 10, 15, 10))); + assertFalse(reader.intersects(geoExtent(25, 10, 25, 10))); } public void testPacManClosedLineString() throws Exception { @@ -188,10 +166,10 @@ public void testPacManClosedLineString() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new GeoExtent(2, -1, 11, 1))); - assertTrue(reader.intersects(new GeoExtent(-12, -12, 12, 12))); - assertTrue(reader.intersects(new GeoExtent(-2, -1, 2, 0))); - assertFalse(reader.intersects(new GeoExtent(-5, -6, 2, -2))); + assertTrue(reader.intersects(geoExtent(2, -1, 11, 1))); + assertTrue(reader.intersects(geoExtent(-12, -12, 12, 12))); + assertTrue(reader.intersects(geoExtent(-2, -1, 2, 0))); + assertFalse(reader.intersects(geoExtent(-5, -6, 2, -2))); } public void testPacManLineString() throws Exception { @@ -205,10 +183,10 @@ public void testPacManLineString() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new GeoExtent(2, -1, 11, 1))); - assertTrue(reader.intersects(new GeoExtent(-12, -12, 12, 12))); - assertTrue(reader.intersects(new GeoExtent(-2, -1, 2, 0))); - assertFalse(reader.intersects(new GeoExtent(-5, -6, 2, -2))); + assertTrue(reader.intersects(geoExtent(2, -1, 11, 1))); + assertTrue(reader.intersects(geoExtent(-12, -12, 12, 12))); + assertTrue(reader.intersects(geoExtent(-2, -1, 2, 0))); + assertFalse(reader.intersects(geoExtent(-5, -6, 2, -2))); } public void testPacManPoints() throws Exception { @@ -239,6 +217,6 @@ public void testPacManPoints() throws Exception { writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(new GeoExtent(xMin, yMin, xMax, yMax))); + assertTrue(reader.intersects(geoExtent(xMin, yMin, xMax, yMax))); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java index 85fae10a9a205..6e8a3d620a67a 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java @@ -18,13 +18,9 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.geo.GeoEncodingUtils; -import org.elasticsearch.common.geo.builders.PointBuilder; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geo.geometry.Point; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.geo.RandomShapeGenerator; import java.io.IOException; import java.nio.ByteBuffer; @@ -34,23 +30,22 @@ public class Point2DTests extends ESTestCase { public void testOnePoint() throws IOException { - PointBuilder gen = (PointBuilder) RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POINT); - Point point = gen.buildGeometry(); - int x = GeoEncodingUtils.encodeLongitude(point.getLon()); - int y = GeoEncodingUtils.encodeLatitude(point.getLat()); + int x = randomIntBetween(-90, 90); + int y = randomIntBetween(-90, 90); Point2DWriter writer = new Point2DWriter(x, y); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); - assertThat(reader.getExtent(), equalTo(new Extent(x, y, x, y))); - assertTrue(reader.intersects(new Extent(x, y, x, y))); - assertTrue(reader.intersects(new Extent(x, y, x + randomIntBetween(1, 10), y + randomIntBetween(1, 10)))); - assertTrue(reader.intersects(new Extent(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), x, y))); - assertTrue(reader.intersects(new Extent(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), + assertThat(reader.getExtent(), equalTo(Extent.fromPoint(x, y))); + assertThat(reader.getExtent(), equalTo(reader.getExtent())); + assertTrue(reader.intersects(Extent.fromPoint(x, y))); + assertTrue(reader.intersects(Extent.fromPoints(x, y, x + randomIntBetween(1, 10), y + randomIntBetween(1, 10)))); + assertTrue(reader.intersects(Extent.fromPoints(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), x, y))); + assertTrue(reader.intersects(Extent.fromPoints(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), x + randomIntBetween(1, 10), y + randomIntBetween(1, 10)))); - assertFalse(reader.intersects(new Extent(x - randomIntBetween(10, 100), y - randomIntBetween(10, 100), + assertFalse(reader.intersects(Extent.fromPoints(x - randomIntBetween(10, 100), y - randomIntBetween(10, 100), x - randomIntBetween(1, 10), y - randomIntBetween(1, 10)))); } @@ -60,7 +55,7 @@ public void testPoints() throws IOException { int maxX = randomIntBetween(minX + 10, 180); int minY = randomIntBetween(-90, 80); int maxY = randomIntBetween(minY + 10, 90); - Extent extent = new Extent(minX, minY, maxX, maxY); + Extent extent = Extent.fromPoints(minX, minY, maxX, maxY); int numPoints = randomIntBetween(2, 1000); int[] x = new int[numPoints]; @@ -75,6 +70,7 @@ public void testPoints() throws IOException { writer.writeTo(output); output.close(); Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + assertThat(reader.getExtent(), equalTo(reader.getExtent())); assertThat(reader.getExtent(), equalTo(writer.getExtent())); assertTrue(reader.intersects(extent)); } From 8d396ab2f459f63d310fd07919ad2a02c11af038 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 9 Aug 2019 15:03:49 -0700 Subject: [PATCH 18/62] support proper ValuesSourceConfig resolution of GEO and GEO_SHAPE. (#45317) * support proper ValuesSourceConfig resolution of GEO and GEO_SHAPE. This change includes introduction of Geo and Geo-Shape ValueSourceType and ValueType. - include missing value resolution - include ability for aggregations to support either geo_point or geo_shape * add docs and rename IndexGeoFieldData to IndexGeometryFieldData --- .../org/elasticsearch/common/geo/Extent.java | 2 +- .../common/geo/GeometryTreeWriter.java | 4 + .../fielddata/IndexGeoPointFieldData.java | 4 +- .../fielddata/IndexGeoShapeFieldData.java | 4 +- .../fielddata/IndexGeometryFieldData.java | 27 +++ .../index/fielddata/MultiGeoValues.java | 23 ++- .../aggregations/support/MissingValues.java | 34 ++-- .../aggregations/support/ValueType.java | 8 +- .../aggregations/support/ValuesSource.java | 50 +++-- .../support/ValuesSourceConfig.java | 36 +++- .../support/ValuesSourceType.java | 3 +- .../metrics/ValueCountAggregatorTests.java | 9 + .../support/ValuesSourceConfigTests.java | 181 ++++++++++++++++++ .../support/ValuesSourceTypeTests.java | 4 + 14 files changed, 342 insertions(+), 47 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/IndexGeometryFieldData.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index ebf2fabbac349..c7f9c69942893 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -58,7 +58,7 @@ public class Extent implements Writeable { * @param y the y-coordinate of the point * @return the extent of the point */ - static Extent fromPoint(int x, int y) { + public static Extent fromPoint(int x, int y) { return new Extent(y, y, x < 0 ? x : Integer.MAX_VALUE, x < 0 ? x : Integer.MIN_VALUE, diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 22e0757b90ff2..4aec7fb406d11 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -53,6 +53,10 @@ public GeometryTreeWriter(Geometry geometry) { geometry.visit(builder); } + public Extent extent() { + return new Extent(builder.top, builder.bottom, builder.negLeft, builder.negRight, builder.posLeft, builder.posRight); + } + @Override public void writeTo(StreamOutput out) throws IOException { // only write a geometry extent for the tree if the tree diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java index 959e3b9ffea26..b44665f1e6f72 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoPointFieldData.java @@ -21,7 +21,7 @@ /** - * Specialization of {@link IndexFieldData} for geo points. + * Specialization of {@link IndexGeometryFieldData} for geo points. */ -public interface IndexGeoPointFieldData extends IndexFieldData { +public interface IndexGeoPointFieldData extends IndexGeometryFieldData { } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java index 841fd1e91d655..c0a6bf5f06f32 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeoShapeFieldData.java @@ -21,7 +21,7 @@ /** - * Specialization of {@link IndexFieldData} for geo points. + * Specialization of {@link IndexGeometryFieldData} for geo shapes. */ -public interface IndexGeoShapeFieldData extends IndexFieldData { +public interface IndexGeoShapeFieldData extends IndexGeometryFieldData { } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeometryFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeometryFieldData.java new file mode 100644 index 0000000000000..ef83bd51a957d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexGeometryFieldData.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.fielddata; + + +/** + * Specialization of {@link IndexFieldData} for geo points and shapes. + */ +public interface IndexGeometryFieldData extends IndexFieldData { +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 38a24643ea039..1347cc143c657 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -18,10 +18,16 @@ */ package org.elasticsearch.index.fielddata; +import org.elasticsearch.common.geo.Extent; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeometryTreeReader; +import org.elasticsearch.common.geo.GeometryTreeWriter; +import org.elasticsearch.geo.geometry.Geometry; +import org.elasticsearch.geo.utils.GeographyValidator; +import org.elasticsearch.geo.utils.WellKnownText; import java.io.IOException; +import java.text.ParseException; /** * A stateful lightweight per document set of geo values. @@ -95,14 +101,19 @@ public String toString() { } public static class GeoShapeValue implements GeoValue { + private static final WellKnownText MISSING_GEOMETRY_PARSER = new WellKnownText(true, new GeographyValidator(true)); + private final GeometryTreeReader reader; + private final Extent extent; public GeoShapeValue(GeometryTreeReader reader) throws IOException { this.reader = reader; + this.extent = reader.getExtent(); } - public GeoShapeValue() { + public GeoShapeValue(Extent extent) { this.reader = null; + this.extent = extent; } @Override @@ -114,6 +125,16 @@ public double lat() { public double lon() { throw new UnsupportedOperationException("centroid of GeoShape is not defined"); } + + public static GeoShapeValue missing(String missing) { + try { + Geometry geometry = MISSING_GEOMETRY_PARSER.fromWKT(missing); + GeometryTreeWriter writer = new GeometryTreeWriter(geometry); + return new GeoShapeValue(writer.extent()); + } catch (IOException | ParseException e) { + throw new IllegalArgumentException("Can't apply missing value [" + missing + "]", e); + } + } } /** diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index 1dcee23906bb7..0a1500ebd9462 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -355,23 +355,6 @@ static LongUnaryOperator getGlobalMapping(SortedSetDocValues values, SortedSetDo } } - public static ValuesSource.GeoPoint replaceMissing(final ValuesSource.GeoPoint valuesSource, - final MultiGeoValues.GeoPointValue missing) { - return new ValuesSource.GeoPoint() { - - @Override - public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return replaceMissing(valuesSource.bytesValues(context), new BytesRef(missing.toString())); - } - - @Override - public MultiGeoValues geoValues(LeafReaderContext context) { - final MultiGeoValues values = valuesSource.geoValues(context); - return replaceMissing(values, missing); - } - }; - } - static MultiGeoValues replaceMissing(final MultiGeoValues values, final MultiGeoValues.GeoValue missing) { return new MultiGeoValues() { @@ -405,6 +388,23 @@ public GeoValue nextValue() throws IOException { }; } + public static ValuesSource.GeoPoint replaceMissing(final ValuesSource.GeoPoint valuesSource, + final MultiGeoValues.GeoPointValue missing) { + return new ValuesSource.GeoPoint() { + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return replaceMissing(valuesSource.bytesValues(context), new BytesRef(missing.toString())); + } + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + final MultiGeoValues values = valuesSource.geoValues(context); + return replaceMissing(values, missing); + } + }; + } + public static ValuesSource.GeoShape replaceMissing(final ValuesSource.GeoShape valuesSource, final MultiGeoValues.GeoShapeValue missing) { return new ValuesSource.GeoShape() { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java index fc23f72eddc9c..d33e74cf637c7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValueType.java @@ -24,7 +24,9 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexGeometryFieldData; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; +import org.elasticsearch.index.fielddata.IndexGeoShapeFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.search.DocValueFormat; @@ -48,7 +50,11 @@ public enum ValueType implements Writeable { // TODO: what is the difference between "number" and "numeric"? NUMERIC((byte) 7, "numeric", "numeric", ValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.RAW), GEOPOINT((byte) 8, "geo_point", "geo_point", ValuesSourceType.GEOPOINT, IndexGeoPointFieldData.class, DocValueFormat.GEOHASH), - BOOLEAN((byte) 9, "boolean", "boolean", ValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.BOOLEAN); + BOOLEAN((byte) 9, "boolean", "boolean", ValuesSourceType.NUMERIC, IndexNumericFieldData.class, DocValueFormat.BOOLEAN), + GEOSHAPE((byte) 10, "geo_shape", "geo_shape", ValuesSourceType.GEOSHAPE, IndexGeoShapeFieldData.class, DocValueFormat.RAW), + // GEO is an abstract ValueType that represents either GEOPOINT or GEOSHAPE in aggregations. It is never directly + // associated with concrete field data + GEO((byte) 11, "geo", "geo", ValuesSourceType.GEO, IndexGeometryFieldData.class, DocValueFormat.RAW); final String description; final ValuesSourceType valuesSourceType; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java index d8436831dd3c0..47ad82b1a2653 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSource.java @@ -486,13 +486,18 @@ public boolean advanceExact(int doc) throws IOException { } } - public interface Geo { - MultiGeoValues geoValues(LeafReaderContext context); - } + /** + * This class represents abstract geo fields that can either be geo_point or geo_shape + */ + public abstract static class Geo extends ValuesSource { + public abstract MultiGeoValues geoValues(LeafReaderContext context); - public abstract static class GeoPoint extends ValuesSource implements Geo { + @Override + public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { + return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoValues(context)); + } - public static final GeoPoint EMPTY = new GeoPoint() { + public static final Geo EMPTY = new Geo() { @Override public MultiGeoValues geoValues(LeafReaderContext context) { @@ -505,12 +510,24 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOExc } }; + } + + public abstract static class GeoPoint extends Geo { + + public static final GeoPoint EMPTY = new GeoPoint() { + + @Override + public MultiGeoValues geoValues(LeafReaderContext context) { + return Geo.EMPTY.geoValues(context); + } + + @Override + public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { + return Geo.EMPTY.bytesValues(context); + } + + }; - @Override - public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { - final MultiGeoValues geoPoints = geoValues(context); - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoPoints); - } public static class Fielddata extends GeoPoint { @@ -531,28 +548,22 @@ public MultiGeoValues geoValues(LeafReaderContext context) { } } - public abstract static class GeoShape extends ValuesSource implements Geo { + public abstract static class GeoShape extends Geo { public static final GeoShape EMPTY = new GeoShape() { @Override public MultiGeoValues geoValues(LeafReaderContext context) { - return org.elasticsearch.index.fielddata.FieldData.emptyMultiGeoValues(); + return Geo.EMPTY.geoValues(context); } @Override public SortedBinaryDocValues bytesValues(LeafReaderContext context) throws IOException { - return org.elasticsearch.index.fielddata.FieldData.emptySortedBinary(); + return Geo.EMPTY.bytesValues(context); } }; - @Override - public DocValueBits docsWithValue(LeafReaderContext context) throws IOException { - final MultiGeoValues geoShapes = geoValues(context); - return org.elasticsearch.index.fielddata.FieldData.docsWithValue(geoShapes); - } - public static class Fielddata extends GeoShape { protected final IndexGeoShapeFieldData indexFieldData; @@ -571,5 +582,4 @@ public MultiGeoValues geoValues(LeafReaderContext context) { } } } - } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index 0aa2af2271c20..591128bab7d49 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.support; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.time.DateFormatter; @@ -103,6 +104,8 @@ public static ValuesSourceConfig resolve( config = new ValuesSourceConfig<>(ValuesSourceType.NUMERIC); } else if (indexFieldData instanceof IndexGeoPointFieldData) { config = new ValuesSourceConfig<>(ValuesSourceType.GEOPOINT); + } else if (indexFieldData instanceof IndexGeoShapeFieldData) { + config = new ValuesSourceConfig<>(ValuesSourceType.GEOSHAPE); } else { config = new ValuesSourceConfig<>(ValuesSourceType.BYTES); } @@ -234,6 +237,7 @@ public VS toValuesSource(QueryShardContext context) { /** Get a value source given its configuration. A return value of null indicates that * no value source could be built. */ @Nullable + @SuppressWarnings("unchecked") public VS toValuesSource(QueryShardContext context, Function resolveMissingAny) { if (!valid()) { throw new IllegalStateException( @@ -247,6 +251,8 @@ public VS toValuesSource(QueryShardContext context, Function 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "geo_point", null, null, null, null); + ValuesSource.GeoPoint valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + MultiGeoValues.GeoValue value = values.nextValue(); + assertThat(value.lat(), closeTo(-10, GeoUtils.TOLERANCE)); + assertThat(value.lon(), closeTo(10, GeoUtils.TOLERANCE)); + } + } + + public void testEmptyGeoPoint() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type", + "geo_point", "type=geo_point"); + client().prepareIndex("index", "type", "1") + .setSource() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher.getIndexReader(), () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "geo_point", null, null, null, null); + ValuesSource.GeoPoint valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertFalse(values.advanceExact(0)); + + config = ValuesSourceConfig.resolve( + context, null, "geo_point", null, "0,0", null, null); + valuesSource = config.toValuesSource(context); + ctx = searcher.getIndexReader().leaves().get(0); + values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + MultiGeoValues.GeoValue value = values.nextValue(); + assertThat(value.lat(), closeTo(0, GeoUtils.TOLERANCE)); + assertThat(value.lon(), closeTo(0, GeoUtils.TOLERANCE)); + } + } + + public void testUnmappedGeoPoint() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type"); + client().prepareIndex("index", "type", "1") + .setSource() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher.getIndexReader(), () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, ValueType.GEOPOINT, "geo_point", null, null, null, null); + ValuesSource.GeoPoint valuesSource = config.toValuesSource(context); + assertNull(valuesSource); + + config = ValuesSourceConfig.resolve( + context, ValueType.GEOPOINT, "geo_point", null, "0,0", null, null); + valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + MultiGeoValues.GeoValue value = values.nextValue(); + assertThat(value.lat(), closeTo(0, GeoUtils.TOLERANCE)); + assertThat(value.lon(), closeTo(0, GeoUtils.TOLERANCE)); + } + } + + public void testGeoShape() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type", + "geo_shape", "type=geo_shape"); + client().prepareIndex("index", "type", "1") + .setSource("geo_shape", "POINT (-10 10)") + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher.getIndexReader(), () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "geo_shape", null, null, null, null); + ValuesSource.GeoShape valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + // TODO (talevy): assert value once BoundingBox is defined +// MultiGeoValues.GeoValue value = values.nextValue(); +// assertThat(value.minX(), closeTo(-10, GeoUtils.TOLERANCE)); +// assertThat(value.minY(), closeTo(10, GeoUtils.TOLERANCE)); + } + } + + public void testEmptyGeoShape() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type", + "geo_shape", "type=geo_shape"); + client().prepareIndex("index", "type", "1") + .setSource() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher.getIndexReader(), () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, null, "geo_shape", null, null, null, null); + ValuesSource.GeoShape valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertFalse(values.advanceExact(0)); + + config = ValuesSourceConfig.resolve( + context, null, "geo_shape", null, "POINT (0 0)", null, null); + valuesSource = config.toValuesSource(context); + ctx = searcher.getIndexReader().leaves().get(0); + values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + // TODO (talevy): assert value once BoundingBox is defined +// MultiGeoValues.GeoValue value = values.nextValue(); +// assertThat(value.minX(), closeTo(-10, GeoUtils.TOLERANCE)); +// assertThat(value.minY(), closeTo(10, GeoUtils.TOLERANCE)); + + IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, + () -> ValuesSourceConfig.resolve(context, ValueType.GEO, "geo_shapes", null, "invalid", + null, null).toValuesSource(context)); + assertThat(exception.getMessage(), equalTo("Unknown geometry type: invalid")); + } + } + + public void testUnmappedGeoShape() throws IOException { + IndexService indexService = createIndex("index", Settings.EMPTY, "type"); + client().prepareIndex("index", "type", "1") + .setSource() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .get(); + + try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { + QueryShardContext context = indexService.newQueryShardContext(0, searcher.getIndexReader(), () -> 42L, null); + + ValuesSourceConfig config = ValuesSourceConfig.resolve( + context, ValueType.GEOSHAPE, "geo_shape", null, null, null, null); + ValuesSource.GeoShape valuesSource = config.toValuesSource(context); + assertNull(valuesSource); + + config = ValuesSourceConfig.resolve( + context, ValueType.GEOSHAPE, "geo_shape", null, "POINT (0 0)", null, null); + valuesSource = config.toValuesSource(context); + LeafReaderContext ctx = searcher.getIndexReader().leaves().get(0); + MultiGeoValues values = valuesSource.geoValues(ctx); + assertTrue(values.advanceExact(0)); + assertEquals(1, values.docValueCount()); + // TODO (talevy): assert value once BoundingBox is defined +// MultiGeoValues.GeoValue value = values.nextValue(); +// assertThat(value.minX(), closeTo(-10, GeoUtils.TOLERANCE)); +// assertThat(value.minY(), closeTo(10, GeoUtils.TOLERANCE)); + } + } + public void testTypeFieldDeprecation() { IndexService indexService = createIndex("index", Settings.EMPTY, "type"); try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java index ad4672b0b49cd..f6213c6077e5f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceTypeTests.java @@ -38,6 +38,7 @@ public void testValidOrdinals() { assertThat(ValuesSourceType.BYTES.ordinal(), equalTo(2)); assertThat(ValuesSourceType.GEOPOINT.ordinal(), equalTo(3)); assertThat(ValuesSourceType.GEOSHAPE.ordinal(), equalTo(4)); + assertThat(ValuesSourceType.GEO.ordinal(), equalTo(5)); } @Override @@ -47,6 +48,7 @@ public void testFromString() { assertThat(ValuesSourceType.fromString("bytes"), equalTo(ValuesSourceType.BYTES)); assertThat(ValuesSourceType.fromString("geopoint"), equalTo(ValuesSourceType.GEOPOINT)); assertThat(ValuesSourceType.fromString("geoshape"), equalTo(ValuesSourceType.GEOSHAPE)); + assertThat(ValuesSourceType.fromString("geo"), equalTo(ValuesSourceType.GEO)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ValuesSourceType.fromString("does_not_exist")); assertThat(e.getMessage(), equalTo("No enum constant org.elasticsearch.search.aggregations.support.ValuesSourceType.DOES_NOT_EXIST")); @@ -60,6 +62,7 @@ public void testReadFrom() throws IOException { assertReadFromStream(2, ValuesSourceType.BYTES); assertReadFromStream(3, ValuesSourceType.GEOPOINT); assertReadFromStream(4, ValuesSourceType.GEOSHAPE); + assertReadFromStream(5, ValuesSourceType.GEO); } @Override @@ -69,5 +72,6 @@ public void testWriteTo() throws IOException { assertWriteToStream(ValuesSourceType.BYTES, 2); assertWriteToStream(ValuesSourceType.GEOPOINT, 3); assertWriteToStream(ValuesSourceType.GEOSHAPE, 4); + assertWriteToStream(ValuesSourceType.GEO, 5); } } From 2fcfbbf6232d4d8511f7d740c1fe8bd7896e25de Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 13 Aug 2019 09:48:01 -0700 Subject: [PATCH 19/62] add geo_shape support to geo_bounds agg (#45413) this commit leverages the new geo_shape doc values and geo values-source-types to add geo_shape support to the geo_bounds aggregation --- .../index/fielddata/MultiGeoValues.java | 62 +++++ .../geogrid/GeoGridAggregationBuilder.java | 2 +- .../range/GeoDistanceAggregationBuilder.java | 2 +- .../metrics/GeoBoundsAggregationBuilder.java | 8 +- .../metrics/GeoBoundsAggregator.java | 36 +-- .../metrics/GeoBoundsAggregatorFactory.java | 6 +- .../GeoCentroidAggregationBuilder.java | 2 +- .../support/ValuesSourceParserHelper.java | 8 +- .../metrics/AbstractGeoTestCase.java | 48 ++-- .../metrics/GeoBoundsAggregatorTests.java | 255 +++++++++++++++++- .../aggregations/metrics/GeoBoundsIT.java | 153 +++++++---- .../aggregations/metrics/GeoCentroidIT.java | 14 +- 12 files changed, 478 insertions(+), 118 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index a653f419d22da..e4ebec0a42736 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.fielddata; +import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.geo.Extent; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeometryTreeReader; @@ -84,6 +85,11 @@ public GeoPoint geoPoint() { return geoPoint; } + @Override + public BoundingBox boundingBox() { + return new BoundingBox(geoPoint); + } + @Override public double lat() { return geoPoint.lat(); @@ -116,6 +122,11 @@ public GeoShapeValue(Extent extent) { this.extent = extent; } + @Override + public BoundingBox boundingBox() { + return new BoundingBox(extent); + } + @Override public double lat() { throw new UnsupportedOperationException("centroid of GeoShape is not defined"); @@ -144,5 +155,56 @@ public static GeoShapeValue missing(String missing) { public interface GeoValue { double lat(); double lon(); + BoundingBox boundingBox(); + } + + public static class BoundingBox { + public final double top; + public final double bottom; + public final double negLeft; + public final double negRight; + public final double posLeft; + public final double posRight; + + BoundingBox(Extent extent) { + this.top = GeoEncodingUtils.decodeLatitude(extent.top); + this.bottom = GeoEncodingUtils.decodeLatitude(extent.bottom); + if (extent.negLeft == Integer.MAX_VALUE) { + this.negLeft = Double.POSITIVE_INFINITY; + } else { + this.negLeft = GeoEncodingUtils.decodeLongitude(extent.negLeft); + } + if (extent.negRight == Integer.MIN_VALUE) { + this.negRight = Double.NEGATIVE_INFINITY; + } else { + this.negRight = GeoEncodingUtils.decodeLongitude(extent.negRight); + } + if (extent.posLeft == Integer.MAX_VALUE) { + this.posLeft = Double.POSITIVE_INFINITY; + } else { + this.posLeft = GeoEncodingUtils.decodeLongitude(extent.posLeft); + } + if (extent.posRight == Integer.MIN_VALUE) { + this.posRight = Double.NEGATIVE_INFINITY; + } else { + this.posRight = GeoEncodingUtils.decodeLongitude(extent.posRight); + } + } + + BoundingBox(GeoPoint point) { + this.top = point.lat(); + this.bottom = point.lat(); + if (point.lon() < 0) { + this.negLeft = point.lon(); + this.negRight = point.lon(); + this.posLeft = Double.POSITIVE_INFINITY; + this.posRight = Double.NEGATIVE_INFINITY; + } else { + this.negLeft = Double.POSITIVE_INFINITY; + this.negRight = Double.NEGATIVE_INFINITY; + this.posLeft = point.lon(); + this.posRight = point.lon(); + } + } } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index bae95c84c00ed..f8e8b5823361b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -61,7 +61,7 @@ protected interface PrecisionParser { public static ObjectParser createParser(String name, PrecisionParser precisionParser) { ObjectParser parser = new ObjectParser<>(name); - ValuesSourceParserHelper.declareGeoFields(parser, false, false); + ValuesSourceParserHelper.declareGeoPointFields(parser, false, false); parser.declareField((p, builder, context) -> builder.precision(precisionParser.parse(p)), FIELD_PRECISION, org.elasticsearch.common.xcontent.ObjectParser.ValueType.INT); parser.declareInt(GeoGridAggregationBuilder::size, FIELD_SIZE); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java index d9e29d0df469e..8fff7296d69db 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/range/GeoDistanceAggregationBuilder.java @@ -60,7 +60,7 @@ public class GeoDistanceAggregationBuilder extends ValuesSourceAggregationBuilde private static final ObjectParser PARSER; static { PARSER = new ObjectParser<>(GeoDistanceAggregationBuilder.NAME); - ValuesSourceParserHelper.declareGeoFields(PARSER, true, false); + ValuesSourceParserHelper.declareGeoPointFields(PARSER, true, false); PARSER.declareBoolean(GeoDistanceAggregationBuilder::keyed, RangeAggregator.KEYED_FIELD); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java index 6f6101fc45ee4..5af442b0117ad 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregationBuilder.java @@ -39,7 +39,7 @@ import java.util.Map; import java.util.Objects; -public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder { +public class GeoBoundsAggregationBuilder extends ValuesSourceAggregationBuilder { public static final String NAME = "geo_bounds"; private static final ObjectParser PARSER; @@ -56,7 +56,7 @@ public static AggregationBuilder parse(String aggregationName, XContentParser pa private boolean wrapLongitude = true; public GeoBoundsAggregationBuilder(String name) { - super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(name, ValuesSourceType.GEO, ValueType.GEO); } protected GeoBoundsAggregationBuilder(GeoBoundsAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -73,7 +73,7 @@ protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map config, + protected GeoBoundsAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new GeoBoundsAggregatorFactory(name, config, wrapLongitude, context, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java index 36642f27a4fcf..39aac155828b3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregator.java @@ -41,7 +41,7 @@ final class GeoBoundsAggregator extends MetricsAggregator { static final ParseField WRAP_LONGITUDE_FIELD = new ParseField("wrap_longitude"); - private final ValuesSource.GeoPoint valuesSource; + private final ValuesSource.Geo valuesSource; private final boolean wrapLongitude; DoubleArray tops; DoubleArray bottoms; @@ -51,7 +51,7 @@ final class GeoBoundsAggregator extends MetricsAggregator { DoubleArray negRights; GeoBoundsAggregator(String name, SearchContext aggregationContext, Aggregator parent, - ValuesSource.GeoPoint valuesSource, boolean wrapLongitude, List pipelineAggregators, + ValuesSource.Geo valuesSource, boolean wrapLongitude, List pipelineAggregators, Map metaData) throws IOException { super(name, aggregationContext, parent, pipelineAggregators, metaData); this.valuesSource = valuesSource; @@ -105,31 +105,13 @@ public void collect(int doc, long bucket) throws IOException { for (int i = 0; i < valuesCount; ++i) { MultiGeoValues.GeoValue value = values.nextValue(); - - double top = tops.get(bucket); - if (value.lat() > top) { - top = value.lat(); - } - double bottom = bottoms.get(bucket); - if (value.lat() < bottom) { - bottom = value.lat(); - } - double posLeft = posLefts.get(bucket); - if (value.lon() >= 0 && value.lon() < posLeft) { - posLeft = value.lon(); - } - double posRight = posRights.get(bucket); - if (value.lon() >= 0 && value.lon() > posRight) { - posRight = value.lon(); - } - double negLeft = negLefts.get(bucket); - if (value.lon() < 0 && value.lon() < negLeft) { - negLeft = value.lon(); - } - double negRight = negRights.get(bucket); - if (value.lon() < 0 && value.lon() > negRight) { - negRight = value.lon(); - } + MultiGeoValues.BoundingBox bounds = value.boundingBox(); + double top = Math.max(tops.get(bucket), bounds.top); + double bottom = Math.min(bottoms.get(bucket), bounds.bottom); + double posLeft = Math.min(posLefts.get(bucket), bounds.posLeft); + double posRight = Math.max(posRights.get(bucket), bounds.posRight); + double negLeft = Math.min(negLefts.get(bucket), bounds.negLeft); + double negRight = Math.max(negRights.get(bucket), bounds.negRight); tops.set(bucket, top); bottoms.set(bucket, bottom); posLefts.set(bucket, posLeft); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java index de8936079c236..5e8df7829b656 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorFactory.java @@ -32,11 +32,11 @@ import java.util.List; import java.util.Map; -class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory { +class GeoBoundsAggregatorFactory extends ValuesSourceAggregatorFactory { private final boolean wrapLongitude; - GeoBoundsAggregatorFactory(String name, ValuesSourceConfig config, boolean wrapLongitude, + GeoBoundsAggregatorFactory(String name, ValuesSourceConfig config, boolean wrapLongitude, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { super(name, config, context, parent, subFactoriesBuilder, metaData); @@ -50,7 +50,7 @@ protected Aggregator createUnmapped(Aggregator parent, List } @Override - protected Aggregator doCreateInternal(ValuesSource.GeoPoint valuesSource, Aggregator parent, boolean collectsFromSingleBucket, + protected Aggregator doCreateInternal(ValuesSource.Geo valuesSource, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { return new GeoBoundsAggregator(name, context, parent, valuesSource, wrapLongitude, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java index 98e8f2e9dbfbe..af2d1a600243a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java @@ -45,7 +45,7 @@ public class GeoCentroidAggregationBuilder private static final ObjectParser PARSER; static { PARSER = new ObjectParser<>(GeoCentroidAggregationBuilder.NAME); - ValuesSourceParserHelper.declareGeoFields(PARSER, true, false); + ValuesSourceParserHelper.declareGeoPointFields(PARSER, true, false); } public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java index 24bdffaa3fa89..0e0c17fb0aad1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceParserHelper.java @@ -51,12 +51,18 @@ public static void declareBytesFields( declareFields(objectParser, scriptable, formattable, false, ValueType.STRING); } - public static void declareGeoFields( + public static void declareGeoPointFields( AbstractObjectParser, T> objectParser, boolean scriptable, boolean formattable) { declareFields(objectParser, scriptable, formattable, false, ValueType.GEOPOINT); } + public static void declareGeoFields( + AbstractObjectParser, T> objectParser, + boolean scriptable, boolean formattable) { + declareFields(objectParser, scriptable, formattable, false, ValueType.GEO); + } + private static void declareFields( AbstractObjectParser, T> objectParser, boolean scriptable, boolean formattable, boolean timezoneAware, ValueType targetValueType) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java index 4e58d836683e7..5d6c10079d145 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -23,7 +23,6 @@ import com.carrotsearch.hppc.ObjectIntMap; import com.carrotsearch.hppc.ObjectObjectHashMap; import com.carrotsearch.hppc.ObjectObjectMap; - import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Strings; @@ -34,6 +33,9 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; @@ -51,7 +53,8 @@ @ESIntegTestCase.SuiteScopeTestCase public abstract class AbstractGeoTestCase extends ESIntegTestCase { - protected static final String SINGLE_VALUED_FIELD_NAME = "geo_value"; + protected static final String SINGLE_VALUED_GEOPOINT_FIELD_NAME = "geopoint_value"; + protected static final String SINGLE_VALUED_GEOSHAPE_FIELD_NAME = "geoshape_value"; protected static final String MULTI_VALUED_FIELD_NAME = "geo_values"; protected static final String NUMBER_FIELD_NAME = "l_values"; protected static final String UNMAPPED_IDX_NAME = "idx_unmapped"; @@ -74,7 +77,7 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase { public void setupSuiteScopeCluster() throws Exception { createIndex(UNMAPPED_IDX_NAME); assertAcked(prepareCreate(IDX_NAME) - .addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point", + .addMapping("type", SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=keyword")); singleTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); @@ -117,7 +120,7 @@ public void setupSuiteScopeCluster() throws Exception { multiVal[1] = multiValues[(i+1) % numUniqueGeoPoints]; builders.add(client().prepareIndex(IDX_NAME, "type").setSource(jsonBuilder() .startObject() - .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, singleVal.lon(), singleVal.lat()) .startArray(MULTI_VALUED_FIELD_NAME) .startArray().value(multiVal[0].lon()).value(multiVal[0].lat()).endArray() .startArray().value(multiVal[1].lon()).value(multiVal[1].lat()).endArray() @@ -133,31 +136,36 @@ public void setupSuiteScopeCluster() throws Exception { multiCentroid.lon() + (newMVLon - multiCentroid.lon()) / (i+1)); } - assertAcked(prepareCreate(EMPTY_IDX_NAME).addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + assertAcked(prepareCreate(EMPTY_IDX_NAME).addMapping("type", SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point")); assertAcked(prepareCreate(DATELINE_IDX_NAME) - .addMapping("type", SINGLE_VALUED_FIELD_NAME, - "type=geo_point", MULTI_VALUED_FIELD_NAME, - "type=geo_point", NUMBER_FIELD_NAME, - "type=long", "tag", "type=keyword")); + .addMapping("type", + SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point", + SINGLE_VALUED_GEOSHAPE_FIELD_NAME, "type=geo_shape", + MULTI_VALUED_FIELD_NAME, "type=geo_point", + NUMBER_FIELD_NAME, "type=long", + "tag", "type=keyword")); - GeoPoint[] geoValues = new GeoPoint[5]; - geoValues[0] = new GeoPoint(38, 178); - geoValues[1] = new GeoPoint(12, -179); - geoValues[2] = new GeoPoint(-24, 170); - geoValues[3] = new GeoPoint(32, -175); - geoValues[4] = new GeoPoint(-11, 178); + GeoPoint[] geoPointValues = new GeoPoint[5]; + geoPointValues[0] = new GeoPoint(38, 178); + geoPointValues[1] = new GeoPoint(12, -179); + geoPointValues[2] = new GeoPoint(-24, 170); + geoPointValues[3] = new GeoPoint(32, -175); + geoPointValues[4] = new GeoPoint(-11, 178); + Line line = new Line(new double[] { 178, -179, 170, -175, 178 }, new double[] { 38, 12, -24, 32, -11 }); + String lineAsWKT = new WellKnownText(false, new GeographyValidator(true)).toWKT(line); for (int i = 0; i < 5; i++) { builders.add(client().prepareIndex(DATELINE_IDX_NAME, "type").setSource(jsonBuilder() .startObject() - .array(SINGLE_VALUED_FIELD_NAME, geoValues[i].lon(), geoValues[i].lat()) + .array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, geoPointValues[i].lon(), geoPointValues[i].lat()) + .field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME, lineAsWKT) .field(NUMBER_FIELD_NAME, i) .field("tag", "tag" + i) .endObject())); } assertAcked(prepareCreate(HIGH_CARD_IDX_NAME).setSettings(Settings.builder().put("number_of_shards", 2)) - .addMapping("type", SINGLE_VALUED_FIELD_NAME, + .addMapping("type", SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long,store=true", @@ -167,7 +175,7 @@ public void setupSuiteScopeCluster() throws Exception { singleVal = singleValues[i % numUniqueGeoPoints]; builders.add(client().prepareIndex(HIGH_CARD_IDX_NAME, "type").setSource(jsonBuilder() .startObject() - .array(SINGLE_VALUED_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, singleVal.lon(), singleVal.lat()) .startArray(MULTI_VALUED_FIELD_NAME) .startArray() .value(multiValues[i % numUniqueGeoPoints].lon()) @@ -185,8 +193,8 @@ public void setupSuiteScopeCluster() throws Exception { } builders.add(client().prepareIndex(IDX_ZERO_NAME, "type").setSource( - jsonBuilder().startObject().array(SINGLE_VALUED_FIELD_NAME, 0.0, 1.0).endObject())); - assertAcked(prepareCreate(IDX_ZERO_NAME).addMapping("type", SINGLE_VALUED_FIELD_NAME, "type=geo_point")); + jsonBuilder().startObject().array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, 0.0, 1.0).endObject())); + assertAcked(prepareCreate(IDX_ZERO_NAME).addMapping("type", SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point")); indexRandom(true, builders); ensureSearchable(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java index 562d29416dcd8..163f8c27bdda5 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java @@ -26,18 +26,37 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.test.geo.RandomGeoGenerator; +import java.util.ArrayList; +import java.util.List; + import static org.elasticsearch.search.aggregations.metrics.InternalGeoBoundsTests.GEOHASH_TOLERANCE; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; public class GeoBoundsAggregatorTests extends AggregatorTestCase { - public void testEmpty() throws Exception { + + public void testEmptyGeoPoint() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg") @@ -61,7 +80,31 @@ public void testEmpty() throws Exception { } } - public void testRandom() throws Exception { + public void testEmptyGeoShape() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg") + .field("field") + .wrapLongitude(false); + + MappedFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("field"); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertTrue(Double.isInfinite(bounds.top)); + assertTrue(Double.isInfinite(bounds.bottom)); + assertTrue(Double.isInfinite(bounds.posLeft)); + assertTrue(Double.isInfinite(bounds.posRight)); + assertTrue(Double.isInfinite(bounds.negLeft)); + assertTrue(Double.isInfinite(bounds.negRight)); + assertFalse(AggregationInspectionHelper.hasValue(bounds)); + } + } + } + + public void testRandomPoints() throws Exception { double top = Double.NEGATIVE_INFINITY; double bottom = Double.POSITIVE_INFINITY; double posLeft = Double.POSITIVE_INFINITY; @@ -118,4 +161,212 @@ public void testRandom() throws Exception { } } } + + public void testRandomShapes() throws Exception { + double top = Double.NEGATIVE_INFINITY; + double bottom = Double.POSITIVE_INFINITY; + double posLeft = Double.POSITIVE_INFINITY; + double posRight = Double.NEGATIVE_INFINITY; + double negLeft = Double.POSITIVE_INFINITY; + double negRight = Double.NEGATIVE_INFINITY; + int numDocs = randomIntBetween(50, 100); + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + int numValues = randomIntBetween(1, 5); + List points = new ArrayList<>(); + for (int j = 0; j < numValues; j++) { + GeoPoint point = RandomGeoGenerator.randomPoint(random()); + points.add(new Point(point.getLon(), point.getLat())); + if (point.getLat() > top) { + top = point.getLat(); + } + if (point.getLat() < bottom) { + bottom = point.getLat(); + } + if (point.getLon() >= 0 && point.getLon() < posLeft) { + posLeft = point.getLon(); + } + if (point.getLon() >= 0 && point.getLon() > posRight) { + posRight = point.getLon(); + } + if (point.getLon() < 0 && point.getLon() < negLeft) { + negLeft = point.getLon(); + } + if (point.getLon() < 0 && point.getLon() > negRight) { + negRight = point.getLon(); + } + } + doc.add(new BinaryGeoShapeDocValuesField("field", new MultiPoint(points))); + w.addDocument(doc); + } + GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg") + .field("field") + .wrapLongitude(false); + + MappedFieldType fieldType = new GeoShapeFieldMapper.GeoShapeFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("field"); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoBounds bounds = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertThat(bounds.top, closeTo(top, GEOHASH_TOLERANCE)); + assertThat(bounds.bottom, closeTo(bottom, GEOHASH_TOLERANCE)); + assertThat(bounds.posLeft, closeTo(posLeft, GEOHASH_TOLERANCE)); + assertThat(bounds.posRight, closeTo(posRight, GEOHASH_TOLERANCE)); + assertThat(bounds.negRight, closeTo(negRight, GEOHASH_TOLERANCE)); + assertThat(bounds.negLeft, closeTo(negLeft, GEOHASH_TOLERANCE)); + assertTrue(AggregationInspectionHelper.hasValue(bounds)); + } + } + } + + public void testFiji() throws Exception { + XContentParser fijiParser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + new BytesArray("{\n" + + " \"type\": \"MultiPolygon\",\n" + + " \"coordinates\": [\n" + + " [\n" + + " [\n" + + " [\n" + + " 178.3736,\n" + + " -17.33992\n" + + " ],\n" + + " [\n" + + " 178.71806,\n" + + " -17.62846\n" + + " ],\n" + + " [\n" + + " 178.55271,\n" + + " -18.15059\n" + + " ],\n" + + " [\n" + + " 177.93266,\n" + + " -18.28799\n" + + " ],\n" + + " [\n" + + " 177.38146,\n" + + " -18.16432\n" + + " ],\n" + + " [\n" + + " 177.28504,\n" + + " -17.72465\n" + + " ],\n" + + " [\n" + + " 177.67087,\n" + + " -17.38114\n" + + " ],\n" + + " [\n" + + " 178.12557,\n" + + " -17.50481\n" + + " ],\n" + + " [\n" + + " 178.3736,\n" + + " -17.33992\n" + + " ]\n" + + " ]\n" + + " ],\n" + + " [\n" + + " [\n" + + " [\n" + + " 179.364143,\n" + + " -16.801354\n" + + " ],\n" + + " [\n" + + " 178.725059,\n" + + " -17.012042\n" + + " ],\n" + + " [\n" + + " 178.596839,\n" + + " -16.63915\n" + + " ],\n" + + " [\n" + + " 179.096609,\n" + + " -16.433984\n" + + " ],\n" + + " [\n" + + " 179.413509,\n" + + " -16.379054\n" + + " ],\n" + + " [\n" + + " 180,\n" + + " -16.067133\n" + + " ],\n" + + " [\n" + + " 180,\n" + + " -16.555217\n" + + " ],\n" + + " [\n" + + " 179.364143,\n" + + " -16.801354\n" + + " ]\n" + + " ]\n" + + " ],\n" + + " [\n" + + " [\n" + + " [\n" + + " -179.917369,\n" + + " -16.501783\n" + + " ],\n" + + " [\n" + + " -180,\n" + + " -16.555217\n" + + " ],\n" + + " [\n" + + " -180,\n" + + " -16.067133\n" + + " ],\n" + + " [\n" + + " -179.79332,\n" + + " -16.020882\n" + + " ],\n" + + " [\n" + + " -179.917369,\n" + + " -16.501783\n" + + " ]\n" + + " ]\n" + + " ]\n" + + " ]\n" + + " }"), XContentType.JSON); + + fijiParser.nextToken(); + MultiPolygon fiji = (MultiPolygon) new GeometryParser(true, true, true).parse(fijiParser); + MultiPolygon geometryForIndexing = (MultiPolygon) new GeoShapeIndexer(true, "indexer").prepareForIndexing(fiji); + + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + Document doc = new Document(); + doc.add(new BinaryGeoShapeDocValuesField("fiji_shape", geometryForIndexing)); + for (Polygon poly : fiji) { + for (int i = 0; i < poly.getPolygon().length(); i++) { + doc.add(new LatLonDocValuesField("fiji_points", poly.getPolygon().getLat(i), poly.getPolygon().getLon(i))); + } + } + + w.addDocument(doc); + + boolean wrapLongitude = randomBoolean(); + GeoBoundsAggregationBuilder pointsAggBuilder = new GeoBoundsAggregationBuilder("my_agg") + .field("fiji_points").wrapLongitude(wrapLongitude); + MappedFieldType pointsType = new GeoPointFieldMapper.GeoPointFieldType(); + pointsType.setHasDocValues(true); + pointsType.setName("fiji_points"); + + GeoBoundsAggregationBuilder shapesAggBuilder = new GeoBoundsAggregationBuilder("my_agg") + .field("fiji_shape").wrapLongitude(wrapLongitude); + MappedFieldType shapeType = new GeoShapeFieldMapper.GeoShapeFieldType(); + shapeType.setHasDocValues(true); + shapeType.setName("fiji_shape"); + + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoBounds pointBounds = search(searcher, new MatchAllDocsQuery(), pointsAggBuilder, pointsType); + InternalGeoBounds shapeBounds = search(searcher, new MatchAllDocsQuery(), shapesAggBuilder, shapeType); + assertTrue(AggregationInspectionHelper.hasValue(pointBounds)); + assertTrue(AggregationInspectionHelper.hasValue(shapeBounds)); + assertThat(shapeBounds, equalTo(pointBounds)); + } + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java index baf9aab19dc82..4209fe933c100 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java @@ -19,9 +19,11 @@ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.util.BigArray; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.global.Global; import org.elasticsearch.search.aggregations.bucket.terms.Terms; @@ -34,9 +36,11 @@ import static org.elasticsearch.search.aggregations.AggregationBuilders.geoBounds; import static org.elasticsearch.search.aggregations.AggregationBuilders.global; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertFailures; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; @@ -45,19 +49,20 @@ @ESIntegTestCase.SuiteScopeTestCase public class GeoBoundsIT extends AbstractGeoTestCase { - private static final String aggName = "geoBounds"; + private static final String geoPointAggName = "geoPointBounds"; + private static final String geoShapeAggName = "geoShapeBounds"; public void testSingleValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) .wrapLongitude(false)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); @@ -71,7 +76,8 @@ public void testSingleValuedField_getProperty() throws Exception { .prepareSearch(IDX_NAME) .setQuery(matchAllQuery()) .addAggregation( - global("global").subAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false))) + global("global").subAggregation(geoBounds(geoPointAggName) + .field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(false))) .get(); assertSearchResponse(searchResponse); @@ -83,36 +89,38 @@ public void testSingleValuedField_getProperty() throws Exception { assertThat(global.getAggregations(), notNullValue()); assertThat(global.getAggregations().asMap().size(), equalTo(1)); - GeoBounds geobounds = global.getAggregations().get(aggName); + GeoBounds geobounds = global.getAggregations().get(geoPointAggName); assertThat(geobounds, notNullValue()); - assertThat(geobounds.getName(), equalTo(aggName)); - assertThat((GeoBounds) ((InternalAggregation)global).getProperty(aggName), sameInstance(geobounds)); + assertThat(geobounds.getName(), equalTo(geoPointAggName)); + assertThat((GeoBounds) ((InternalAggregation)global).getProperty(geoPointAggName), sameInstance(geobounds)); GeoPoint topLeft = geobounds.topLeft(); GeoPoint bottomRight = geobounds.bottomRight(); assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); - assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".top"), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); - assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".left"), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); - assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".bottom"), + assertThat((double) ((InternalAggregation)global).getProperty(geoPointAggName + ".top"), + closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat((double) ((InternalAggregation)global).getProperty(geoPointAggName + ".left"), + closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat((double) ((InternalAggregation)global).getProperty(geoPointAggName + ".bottom"), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); - assertThat((double) ((InternalAggregation)global).getProperty(aggName + ".right"), + assertThat((double) ((InternalAggregation)global).getProperty(geoPointAggName + ".right"), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); } public void testMultiValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) - .addAggregation(geoBounds(aggName).field(MULTI_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(MULTI_VALUED_FIELD_NAME) .wrapLongitude(false)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(multiTopLeft.lat(), GEOHASH_TOLERANCE)); @@ -123,15 +131,15 @@ public void testMultiValuedField() throws Exception { public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch(UNMAPPED_IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) .wrapLongitude(false)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft, equalTo(null)); @@ -140,15 +148,15 @@ public void testUnmapped() throws Exception { public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME, UNMAPPED_IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) .wrapLongitude(false)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); @@ -160,23 +168,27 @@ public void testPartiallyUnmapped() throws Exception { public void testEmptyAggregation() throws Exception { SearchResponse searchResponse = client().prepareSearch(EMPTY_IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) + .wrapLongitude(false)) + .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME) .wrapLongitude(false)) .get(); - assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); - GeoBounds geoBounds = searchResponse.getAggregations().get(aggName); - assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); - GeoPoint topLeft = geoBounds.topLeft(); - GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft, equalTo(null)); - assertThat(bottomRight, equalTo(null)); + for (String aggName : List.of(geoPointAggName, geoShapeAggName)) { + assertThat(searchResponse.getHits().getTotalHits().value, equalTo(0L)); + GeoBounds geoBounds = searchResponse.getAggregations().get(aggName); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo(aggName)); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft, equalTo(null)); + assertThat(bottomRight, equalTo(null)); + } } public void testSingleValuedFieldNearDateLine() throws Exception { SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) .wrapLongitude(false)) .get(); @@ -185,9 +197,9 @@ public void testSingleValuedFieldNearDateLine() throws Exception { GeoPoint geoValuesTopLeft = new GeoPoint(38, -179); GeoPoint geoValuesBottomRight = new GeoPoint(-24, 178); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); @@ -197,24 +209,26 @@ public void testSingleValuedFieldNearDateLine() throws Exception { } public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { - GeoPoint geoValuesTopLeft = new GeoPoint(38, 170); GeoPoint geoValuesBottomRight = new GeoPoint(-24, -175); SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(true)) + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(true)) + .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME).wrapLongitude(true)) .get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); - assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); - GeoPoint topLeft = geoBounds.topLeft(); - GeoPoint bottomRight = geoBounds.bottomRight(); - assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); - assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); - assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); - assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); + for (String aggName : List.of(geoPointAggName, geoShapeAggName)) { + GeoBounds geoBounds = response.getAggregations().get(aggName); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo(aggName)); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); + } } /** @@ -222,8 +236,8 @@ public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { */ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) - .addAggregation(terms("terms").field(NUMBER_FIELD_NAME).subAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME) - .wrapLongitude(false))) + .addAggregation(terms("terms").field(NUMBER_FIELD_NAME).subAggregation(geoBounds(geoPointAggName) + .field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(false))) .get(); assertSearchResponse(response); @@ -237,9 +251,9 @@ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { Bucket bucket = buckets.get(i); assertThat(bucket, notNullValue()); assertThat("InternalBucket " + bucket.getKey() + " has wrong number of documents", bucket.getDocCount(), equalTo(1L)); - GeoBounds geoBounds = bucket.getAggregations().get(aggName); + GeoBounds geoBounds = bucket.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); assertThat(geoBounds.topLeft().getLat(), allOf(greaterThanOrEqualTo(-90.0), lessThanOrEqualTo(90.0))); assertThat(geoBounds.topLeft().getLon(), allOf(greaterThanOrEqualTo(-180.0), lessThanOrEqualTo(180.0))); assertThat(geoBounds.bottomRight().getLat(), allOf(greaterThanOrEqualTo(-90.0), lessThanOrEqualTo(90.0))); @@ -249,13 +263,13 @@ public void testSingleValuedFieldAsSubAggToHighCardTermsAgg() { public void testSingleValuedFieldWithZeroLon() throws Exception { SearchResponse response = client().prepareSearch(IDX_ZERO_NAME) - .addAggregation(geoBounds(aggName).field(SINGLE_VALUED_FIELD_NAME).wrapLongitude(false)).get(); + .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(false)).get(); assertSearchResponse(response); - GeoBounds geoBounds = response.getAggregations().get(aggName); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); @@ -263,4 +277,41 @@ public void testSingleValuedFieldWithZeroLon() throws Exception { assertThat(bottomRight.lat(), closeTo(1.0, GEOHASH_TOLERANCE)); assertThat(bottomRight.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); } + + public void testIncorrectFieldType() { + SearchRequestBuilder searchWithKeywordField = client().prepareSearch(DATELINE_IDX_NAME) + .addAggregation(geoBounds("agg").field("tag")); + assertFailures(searchWithKeywordField, RestStatus.BAD_REQUEST, + containsString("Expected geo_point or geo_shape type on field [tag], but got [keyword]")); + + { + SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) + .addAggregation(geoBounds("agg").missing(randomBoolean() ? "0,0" : "POINT (0 0)").field("non_existent")).get(); + assertSearchResponse(response); + GeoBounds geoBounds = response.getAggregations().get("agg"); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo("agg")); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(0.0, GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(0.0, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(0.0, GEOHASH_TOLERANCE)); + } + + { + SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) + .addAggregation(geoBounds("agg").missing("LINESTRING (30 10, 10 30, 40 40)").field("non_existent")).get(); + assertSearchResponse(response); + GeoBounds geoBounds = response.getAggregations().get("agg"); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo("agg")); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(40, GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(10, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(10, GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(40, GEOHASH_TOLERANCE)); + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java index 9e9af4e65066f..d941ce27ede2f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java @@ -48,7 +48,7 @@ public class GeoCentroidIT extends AbstractGeoTestCase { public void testEmptyAggregation() throws Exception { SearchResponse response = client().prepareSearch(EMPTY_IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME)) .get(); assertSearchResponse(response); @@ -63,7 +63,7 @@ public void testEmptyAggregation() throws Exception { public void testUnmapped() throws Exception { SearchResponse response = client().prepareSearch(UNMAPPED_IDX_NAME) - .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME)) .get(); assertSearchResponse(response); @@ -77,7 +77,7 @@ public void testUnmapped() throws Exception { public void testPartiallyUnmapped() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME, UNMAPPED_IDX_NAME) - .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME)) .get(); assertSearchResponse(response); @@ -93,7 +93,7 @@ public void testPartiallyUnmapped() throws Exception { public void testSingleValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME)) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME)) .get(); assertSearchResponse(response); @@ -109,7 +109,7 @@ public void testSingleValuedField() throws Exception { public void testSingleValueFieldGetProperty() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) .setQuery(matchAllQuery()) - .addAggregation(global("global").subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME))) + .addAggregation(global("global").subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME))) .get(); assertSearchResponse(response); @@ -154,8 +154,8 @@ public void testMultiValuedField() throws Exception { public void testSingleValueFieldAsSubAggToGeohashGrid() throws Exception { SearchResponse response = client().prepareSearch(HIGH_CARD_IDX_NAME) - .addAggregation(geohashGrid("geoGrid").field(SINGLE_VALUED_FIELD_NAME) - .subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_FIELD_NAME))) + .addAggregation(geohashGrid("geoGrid").field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) + .subAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME))) .get(); assertSearchResponse(response); From 04224e9ad5d82c24f51879ccd525d858b6649a24 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 30 Aug 2019 14:22:09 -0700 Subject: [PATCH 20/62] generalize encoding/decoding of coordinates to helper classes (#46154) It is important that the GeometryTreeReader/Writers support different encodings more naturally. As of now, there is a need for GeoEncodingUtil encoding/decoding and XYEncodingUtil encoding/decoding. This PR achieves three goals - abstract the encoding of double-valued x/y coordinates to the int-values that are used when serializing - make the GeometryTree structures re-usable between shape and geo_shape types - have the tree structures take in the raw coordinate values so that more accurate centroid calculations can be done at index-time for the geo-centroid aggregation This PR leaves one open question / leak in the model - the Extent that is queried and returned by these tree structures is still in the encoded form. The reason this is left out of the PR is because Extent is an object that is repetitively used in recursive checks of bounds, and re-encoding the coordinates every time can be costly. - The idea is that the aggregations that rely on intersects queries will convert the BoundingBox structures to Extent objects. This would be done in the future by aggregations like geo*grid. This PR also leaves out an implementation of ShapeCoordinateEncoder that would be used by the shape field-type. This can be added in the future. --- .../common/geo/CoordinateEncoder.java | 31 ++++++ .../common/geo/EdgeTreeReader.java | 19 ++-- .../common/geo/EdgeTreeWriter.java | 41 ++++---- .../common/geo/GeoShapeCoordinateEncoder.java | 58 +++++++++++ .../common/geo/GeometryTreeReader.java | 2 +- .../common/geo/GeometryTreeWriter.java | 73 +++++--------- .../common/geo/Point2DWriter.java | 44 +++++---- .../common/geo/PolygonTreeWriter.java | 6 +- .../index/fielddata/MultiGeoValues.java | 21 ++-- .../mapper/BinaryGeoShapeDocValuesField.java | 3 +- .../common/geo/EdgeTreeTests.java | 44 ++++----- .../geo/GeoShapeCoordinateEncoderTests.java | 65 +++++++++++++ .../common/geo/GeometryTreeTests.java | 95 +++++++++---------- .../common/geo/Point2DTests.java | 8 +- .../common/geo/TestCoordinateEncoder.java | 48 ++++++++++ 15 files changed, 368 insertions(+), 190 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/CoordinateEncoder.java create mode 100644 server/src/main/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoder.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoderTests.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/TestCoordinateEncoder.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/CoordinateEncoder.java b/server/src/main/java/org/elasticsearch/common/geo/CoordinateEncoder.java new file mode 100644 index 0000000000000..c1be8545f87d8 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/CoordinateEncoder.java @@ -0,0 +1,31 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +/** + * Interface for classes that help encode double-valued spatial coordinates x/y to + * their integer-encoded serialized form and decode them back + */ +public interface CoordinateEncoder { + int encodeX(double x); + int encodeY(double y); + double decodeX(int x); + double decodeY(int y); +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index 6d36506ad8ed9..beff33b761b97 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -19,7 +19,6 @@ package org.elasticsearch.common.geo; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; -import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; import java.util.Optional; @@ -57,16 +56,14 @@ public boolean intersects(Extent extent) throws IOException { } } - static Optional checkExtent(StreamInput input, Extent extent) throws IOException { - Extent edgeExtent = new Extent(input); - - if (edgeExtent.minY() > extent.maxY() || edgeExtent.maxX() < extent.minX() - || edgeExtent.maxY() < extent.minY() || edgeExtent.minX() > extent.maxX()) { + static Optional checkExtent(Extent treeExtent, Extent extent) throws IOException { + if (treeExtent.minY() > extent.maxY() || treeExtent.maxX() < extent.minX() + || treeExtent.maxY() < extent.minY() || treeExtent.minX() > extent.maxX()) { return Optional.of(false); // tree and bbox-query are disjoint } - if (extent.minX() <= edgeExtent.minX() && extent.minY() <= edgeExtent.minY() - && extent.maxX() >= edgeExtent.maxX() && extent.maxY() >= edgeExtent.maxY()) { + if (extent.minX() <= treeExtent.minX() && extent.minY() <= treeExtent.minY() + && extent.maxX() >= treeExtent.maxX() && extent.maxY() >= treeExtent.maxY()) { return Optional.of(true); // bbox-query fully contains tree's extent. } return Optional.empty(); @@ -75,7 +72,7 @@ static Optional checkExtent(StreamInput input, Extent extent) throws IO boolean containsBottomLeft(Extent extent) throws IOException { resetInputPosition(); - Optional extentCheck = checkExtent(input, extent); + Optional extentCheck = checkExtent(new Extent(input), extent); if (extentCheck.isPresent()) { return extentCheck.get(); } @@ -92,7 +89,7 @@ boolean containsFully(Extent extent) throws IOException { public boolean crosses(Extent extent) throws IOException { resetInputPosition(); - Optional extentCheck = checkExtent(input, extent); + Optional extentCheck = checkExtent(new Extent(input), extent); if (extentCheck.isPresent()) { return extentCheck.get(); } @@ -100,7 +97,7 @@ public boolean crosses(Extent extent) throws IOException { return crosses(readRoot(input.position()), extent); } - public Edge readRoot(int position) throws IOException { + private Edge readRoot(int position) throws IOException { input.position(position); if (input.readBoolean()) { return readEdge(input.position()); diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index aeca8a793b884..2fb2f1ef230fe 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -43,29 +43,30 @@ public class EdgeTreeWriter extends ShapeTreeWriter { /** - * @param x array of the x-coordinate of points. - * @param y array of the y-coordinate of points. + * @param x array of the x-coordinate of points. + * @param y array of the y-coordinate of points. + * @param coordinateEncoder class that encodes from real-valued x/y to serialized integer coordinate values. */ - EdgeTreeWriter(int[] x, int[] y) { - this(Collections.singletonList(x), Collections.singletonList(y)); + EdgeTreeWriter(double[] x, double[] y, CoordinateEncoder coordinateEncoder) { + this(Collections.singletonList(x), Collections.singletonList(y), coordinateEncoder); } - EdgeTreeWriter(List x, List y) { + EdgeTreeWriter(List x, List y, CoordinateEncoder coordinateEncoder) { this.numShapes = x.size(); - int top = Integer.MIN_VALUE; - int bottom = Integer.MAX_VALUE; - int negLeft = Integer.MAX_VALUE; - int negRight = Integer.MIN_VALUE; - int posLeft = Integer.MAX_VALUE; - int posRight = Integer.MIN_VALUE; + double top = Double.NEGATIVE_INFINITY; + double bottom = Double.POSITIVE_INFINITY; + double negLeft = Double.POSITIVE_INFINITY; + double negRight = Double.NEGATIVE_INFINITY; + double posLeft = Double.POSITIVE_INFINITY; + double posRight = Double.NEGATIVE_INFINITY; List edges = new ArrayList<>(); for (int i = 0; i < y.size(); i++) { for (int j = 1; j < y.get(i).length; j++) { - int y1 = y.get(i)[j - 1]; - int x1 = x.get(i)[j - 1]; - int y2 = y.get(i)[j]; - int x2 = x.get(i)[j]; - int edgeMinY, edgeMaxY; + double y1 = y.get(i)[j - 1]; + double x1 = x.get(i)[j - 1]; + double y2 = y.get(i)[j]; + double x2 = x.get(i)[j]; + double edgeMinY, edgeMaxY; if (y1 < y2) { edgeMinY = y1; edgeMaxY = y2; @@ -73,7 +74,9 @@ public class EdgeTreeWriter extends ShapeTreeWriter { edgeMinY = y2; edgeMaxY = y1; } - edges.add(new Edge(x1, y1, x2, y2, edgeMinY, edgeMaxY)); + edges.add(new Edge(coordinateEncoder.encodeX(x1), coordinateEncoder.encodeY(y1), + coordinateEncoder.encodeX(x2), coordinateEncoder.encodeY(y2), + coordinateEncoder.encodeY(edgeMinY), coordinateEncoder.encodeY(edgeMaxY))); top = Math.max(top, Math.max(y1, y2)); bottom = Math.min(bottom, Math.min(y1, y2)); @@ -108,7 +111,9 @@ public class EdgeTreeWriter extends ShapeTreeWriter { } } edges.sort(Edge::compareTo); - this.extent = new Extent(top, bottom, negLeft, negRight, posLeft, posRight); + this.extent = new Extent(coordinateEncoder.encodeY(top), coordinateEncoder.encodeY(bottom), + coordinateEncoder.encodeX(negLeft), coordinateEncoder.encodeX(negRight), + coordinateEncoder.encodeX(posLeft), coordinateEncoder.encodeX(posRight)); this.tree = createTree(edges, 0, edges.size() - 1); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoder.java b/server/src/main/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoder.java new file mode 100644 index 0000000000000..79666ffb1b1fa --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoder.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +import org.apache.lucene.geo.GeoEncodingUtils; + +public final class GeoShapeCoordinateEncoder implements CoordinateEncoder { + public static final GeoShapeCoordinateEncoder INSTANCE = new GeoShapeCoordinateEncoder(); + + @Override + public int encodeX(double x) { + if (x == Double.NEGATIVE_INFINITY) { + return Integer.MIN_VALUE; + } + if (x == Double.POSITIVE_INFINITY) { + return Integer.MAX_VALUE; + } + return GeoEncodingUtils.encodeLongitude(x); + } + + @Override + public int encodeY(double y) { + if (y == Double.NEGATIVE_INFINITY) { + return Integer.MIN_VALUE; + } + if (y == Double.POSITIVE_INFINITY) { + return Integer.MAX_VALUE; + } + return GeoEncodingUtils.encodeLatitude(y); + } + + @Override + public double decodeX(int x) { + return GeoEncodingUtils.decodeLongitude(x); + } + + @Override + public double decodeY(int y) { + return GeoEncodingUtils.decodeLatitude(y); + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index b73fdc1fed242..9f442d29cc41e 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -56,7 +56,7 @@ public boolean intersects(Extent extent) throws IOException { input.position(0); boolean hasExtent = input.readBoolean(); if (hasExtent) { - Optional extentCheck = EdgeTreeReader.checkExtent(input, extent); + Optional extentCheck = EdgeTreeReader.checkExtent(new Extent(input), extent); if (extentCheck.isPresent()) { return extentCheck.get(); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 85f263e56e9d1..a6c1dfb3a6300 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.geometry.Circle; @@ -48,8 +47,8 @@ public class GeometryTreeWriter implements Writeable { private final GeometryTreeBuilder builder; - public GeometryTreeWriter(Geometry geometry) { - builder = new GeometryTreeBuilder(); + public GeometryTreeWriter(Geometry geometry, CoordinateEncoder coordinateEncoder) { + builder = new GeometryTreeBuilder(coordinateEncoder); geometry.visit(builder); } @@ -77,6 +76,7 @@ public void writeTo(StreamOutput out) throws IOException { class GeometryTreeBuilder implements GeometryVisitor { private List shapeWriters; + private final CoordinateEncoder coordinateEncoder; // integers are used to represent int-encoded lat/lon values int top = Integer.MIN_VALUE; int bottom = Integer.MAX_VALUE; @@ -85,8 +85,9 @@ class GeometryTreeBuilder implements GeometryVisitor { int posLeft = Integer.MAX_VALUE; int posRight = Integer.MIN_VALUE; - GeometryTreeBuilder() { - shapeWriters = new ArrayList<>(); + GeometryTreeBuilder(CoordinateEncoder coordinateEncoder) { + this.coordinateEncoder = coordinateEncoder; + this.shapeWriters = new ArrayList<>(); } private void addWriter(ShapeTreeWriter writer) { @@ -110,20 +111,20 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - addWriter(new EdgeTreeWriter(asLonEncodedArray(line.getLons()), asLatEncodedArray(line.getLats()))); + addWriter(new EdgeTreeWriter(line.getLons(), line.getLats(), coordinateEncoder)); return null; } @Override public Void visit(MultiLine multiLine) { int size = multiLine.size(); - List x = new ArrayList<>(size); - List y = new ArrayList<>(size); + List x = new ArrayList<>(size); + List y = new ArrayList<>(size); for (Line line : multiLine) { - x.add(asLonEncodedArray(line.getLons())); - y.add(asLatEncodedArray(line.getLats())); + x.add(line.getLons()); + y.add(line.getLats()); } - addWriter(new EdgeTreeWriter(x, y)); + addWriter(new EdgeTreeWriter(x, y, coordinateEncoder)); return null; } @@ -131,14 +132,14 @@ public Void visit(MultiLine multiLine) { public Void visit(Polygon polygon) { LinearRing outerShell = polygon.getPolygon(); int numHoles = polygon.getNumberOfHoles(); - List x = new ArrayList<>(numHoles); - List y = new ArrayList<>(numHoles); + List x = new ArrayList<>(numHoles); + List y = new ArrayList<>(numHoles); for (int i = 0; i < numHoles; i++) { LinearRing innerRing = polygon.getHole(i); - x.add(asLonEncodedArray(innerRing.getLons())); - y.add(asLatEncodedArray(innerRing.getLats())); + x.add(innerRing.getLons()); + y.add(innerRing.getLats()); } - addWriter(new PolygonTreeWriter(asLonEncodedArray(outerShell.getLons()), asLatEncodedArray(outerShell.getLats()), x, y)); + addWriter(new PolygonTreeWriter(outerShell.getLons(), outerShell.getLats(), x, y, coordinateEncoder)); return null; } @@ -152,34 +153,28 @@ public Void visit(MultiPolygon multiPolygon) { @Override public Void visit(Rectangle r) { - int encodedMinLat = GeoEncodingUtils.encodeLatitude(r.getMinLat()); - int encodedMaxLat = GeoEncodingUtils.encodeLatitude(r.getMaxLat()); - int encodedMinLon = GeoEncodingUtils.encodeLongitude(r.getMinLon()); - int encodedMaxLon = GeoEncodingUtils.encodeLongitude(r.getMaxLon()); - int[] lats = new int[] { encodedMinLat, encodedMinLat, encodedMaxLat, encodedMaxLat, encodedMinLat }; - int[] lons = new int[] { encodedMinLon, encodedMaxLon, encodedMaxLon, encodedMinLon, encodedMinLon }; - addWriter(new PolygonTreeWriter(lons, lats, Collections.emptyList(), Collections.emptyList())); + double[] lats = new double[] { r.getMinLat(), r.getMinLat(), r.getMaxLat(), r.getMaxLat(), r.getMinLat() }; + double[] lons = new double[] { r.getMinLon(), r.getMaxLon(), r.getMaxLon(), r.getMinLon(), r.getMinLon() }; + addWriter(new PolygonTreeWriter(lons, lats, Collections.emptyList(), Collections.emptyList(), coordinateEncoder)); return null; } @Override public Void visit(Point point) { - int x = GeoEncodingUtils.encodeLongitude(point.getLon()); - int y = GeoEncodingUtils.encodeLatitude(point.getLat()); - Point2DWriter writer = new Point2DWriter(x, y); + Point2DWriter writer = new Point2DWriter(point.getLon(), point.getLat(), coordinateEncoder); addWriter(writer); return null; } @Override public Void visit(MultiPoint multiPoint) { - int[] x = new int[multiPoint.size()]; - int[] y = new int[x.length]; + double[] x = new double[multiPoint.size()]; + double[] y = new double[x.length]; for (int i = 0; i < multiPoint.size(); i++) { - x[i] = GeoEncodingUtils.encodeLongitude(multiPoint.get(i).getLon()); - y[i] = GeoEncodingUtils.encodeLatitude(multiPoint.get(i).getLat()); + x[i] = multiPoint.get(i).getLon(); + y[i] = multiPoint.get(i).getLat(); } - Point2DWriter writer = new Point2DWriter(x, y); + Point2DWriter writer = new Point2DWriter(x, y, coordinateEncoder); addWriter(writer); return null; } @@ -193,21 +188,5 @@ public Void visit(LinearRing ring) { public Void visit(Circle circle) { throw new IllegalArgumentException("invalid shape type found [Circle]"); } - - private int[] asLonEncodedArray(double[] doub) { - int[] intArr = new int[doub.length]; - for (int i = 0; i < intArr.length; i++) { - intArr[i] = GeoEncodingUtils.encodeLongitude(doub[i]); - } - return intArr; - } - - private int[] asLatEncodedArray(double[] doub) { - int[] intArr = new int[doub.length]; - for (int i = 0; i < intArr.length; i++) { - intArr[i] = GeoEncodingUtils.encodeLatitude(doub[i]); - } - return intArr; - } } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java index 4cc182e9dda7b..f59a806f958ab 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java @@ -32,22 +32,24 @@ public class Point2DWriter extends ShapeTreeWriter { private static final int K = 2; private final Extent extent; - private final int[] coords; + private final double[] coords; // size of a leaf node where searches are done sequentially. static final int LEAF_SIZE = 64; + private final CoordinateEncoder coordinateEncoder; - Point2DWriter(int[] x, int[] y) { + Point2DWriter(double[] x, double[] y, CoordinateEncoder coordinateEncoder) { assert x.length == y.length; - int top = Integer.MIN_VALUE; - int bottom = Integer.MAX_VALUE; - int negLeft = Integer.MAX_VALUE; - int negRight = Integer.MIN_VALUE; - int posLeft = Integer.MAX_VALUE; - int posRight = Integer.MIN_VALUE; - coords = new int[x.length * K]; + this.coordinateEncoder = coordinateEncoder; + double top = Double.NEGATIVE_INFINITY; + double bottom = Double.POSITIVE_INFINITY; + double negLeft = Double.POSITIVE_INFINITY; + double negRight = Double.NEGATIVE_INFINITY; + double posLeft = Double.POSITIVE_INFINITY; + double posRight = Double.NEGATIVE_INFINITY; + coords = new double[x.length * K]; for (int i = 0; i < x.length; i++) { - int xi = x[i]; - int yi = y[i]; + double xi = x[i]; + double yi = y[i]; top = Math.max(top, yi); bottom = Math.min(bottom, yi); if (xi >= 0 && xi < posLeft) { @@ -66,12 +68,14 @@ public class Point2DWriter extends ShapeTreeWriter { coords[2 * i + 1] = yi; } sort(0, x.length - 1, 0); - this.extent = new Extent(top, bottom, negLeft, negRight, posLeft, posRight); + this.extent = new Extent(coordinateEncoder.encodeY(top), coordinateEncoder.encodeY(bottom), coordinateEncoder.encodeX(negLeft), + coordinateEncoder.encodeX(negRight), coordinateEncoder.encodeX(posLeft), coordinateEncoder.encodeX(posRight)); } - Point2DWriter(int x, int y) { - coords = new int[] {x, y}; - this.extent = Extent.fromPoint(x, y); + Point2DWriter(double x, double y, CoordinateEncoder coordinateEncoder) { + this.coordinateEncoder = coordinateEncoder; + coords = new double[] {x, y}; + this.extent = Extent.fromPoint(coordinateEncoder.encodeX(x), coordinateEncoder.encodeY(y)); } @Override @@ -91,8 +95,10 @@ public void writeTo(StreamOutput out) throws IOException { if (numPoints > 1) { extent.writeTo(out); } - for (int coord : coords) { - out.writeInt(coord); + for (int i = 0; i < coords.length; i++) { + double coord = coords[i]; + int encodedCoord = i % 2 == 0 ? coordinateEncoder.encodeX(coord) : coordinateEncoder.encodeY(coord); + out.writeInt(encodedCoord); } } @@ -133,7 +139,7 @@ private void select(int left, int right, int k, int depth) { int newRight = Math.min(right, (int) Math.floor(k + (n - i) * s / n + sd)); select(newLeft, newRight, k, depth); } - int t = coords[2 * k + axis]; + double t = coords[2 * k + axis]; int i = left; int j = right; @@ -176,7 +182,7 @@ private void swapPoint(int i, int j) { } private void swap(int i, int j) { - int tmp = coords[i]; + double tmp = coords[i]; coords[i] = coords[j]; coords[j] = tmp; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java index 1db450bd32227..b48ce0e1cb31b 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java @@ -34,9 +34,9 @@ public class PolygonTreeWriter extends ShapeTreeWriter { private final EdgeTreeWriter outerShell; private final EdgeTreeWriter holes; - public PolygonTreeWriter(int[] x, int[] y, List holesX, List holesY) { - outerShell = new EdgeTreeWriter(x, y); - holes = holesX.isEmpty() ? null : new EdgeTreeWriter(holesX, holesY); + public PolygonTreeWriter(double[] x, double[] y, List holesX, List holesY, CoordinateEncoder coordinateEncoder) { + outerShell = new EdgeTreeWriter(x, y, coordinateEncoder); + holes = holesX.isEmpty() ? null : new EdgeTreeWriter(holesX, holesY, coordinateEncoder); } public Extent getExtent() { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index e4ebec0a42736..6486c74c7fb2a 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -18,9 +18,10 @@ */ package org.elasticsearch.index.fielddata; -import org.apache.lucene.geo.GeoEncodingUtils; +import org.elasticsearch.common.geo.CoordinateEncoder; import org.elasticsearch.common.geo.Extent; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.GeometryTreeReader; import org.elasticsearch.common.geo.GeometryTreeWriter; import org.elasticsearch.geometry.Geometry; @@ -124,7 +125,7 @@ public GeoShapeValue(Extent extent) { @Override public BoundingBox boundingBox() { - return new BoundingBox(extent); + return new BoundingBox(extent, GeoShapeCoordinateEncoder.INSTANCE); } @Override @@ -140,7 +141,7 @@ public double lon() { public static GeoShapeValue missing(String missing) { try { Geometry geometry = MISSING_GEOMETRY_PARSER.fromWKT(missing); - GeometryTreeWriter writer = new GeometryTreeWriter(geometry); + GeometryTreeWriter writer = new GeometryTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); return new GeoShapeValue(writer.extent()); } catch (IOException | ParseException e) { throw new IllegalArgumentException("Can't apply missing value [" + missing + "]", e); @@ -166,28 +167,28 @@ public static class BoundingBox { public final double posLeft; public final double posRight; - BoundingBox(Extent extent) { - this.top = GeoEncodingUtils.decodeLatitude(extent.top); - this.bottom = GeoEncodingUtils.decodeLatitude(extent.bottom); + BoundingBox(Extent extent, CoordinateEncoder coordinateEncoder) { + this.top = coordinateEncoder.decodeY(extent.top); + this.bottom = coordinateEncoder.decodeY(extent.bottom); if (extent.negLeft == Integer.MAX_VALUE) { this.negLeft = Double.POSITIVE_INFINITY; } else { - this.negLeft = GeoEncodingUtils.decodeLongitude(extent.negLeft); + this.negLeft = coordinateEncoder.decodeX(extent.negLeft); } if (extent.negRight == Integer.MIN_VALUE) { this.negRight = Double.NEGATIVE_INFINITY; } else { - this.negRight = GeoEncodingUtils.decodeLongitude(extent.negRight); + this.negRight = coordinateEncoder.decodeX(extent.negRight); } if (extent.posLeft == Integer.MAX_VALUE) { this.posLeft = Double.POSITIVE_INFINITY; } else { - this.posLeft = GeoEncodingUtils.decodeLongitude(extent.posLeft); + this.posLeft = coordinateEncoder.decodeX(extent.posLeft); } if (extent.posRight == Integer.MIN_VALUE) { this.posRight = Double.NEGATIVE_INFINITY; } else { - this.posRight = GeoEncodingUtils.decodeLongitude(extent.posRight); + this.posRight = coordinateEncoder.decodeX(extent.posRight); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java index e81fc7e43abc5..619c7272b4fa7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java @@ -20,6 +20,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.GeometryTreeWriter; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geometry.Geometry; @@ -52,7 +53,7 @@ public BytesRef binaryValue() { } else { geometry = geometries.get(0); } - final GeometryTreeWriter writer = new GeometryTreeWriter(geometry); + final GeometryTreeWriter writer = new GeometryTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); return output.bytes().toBytesRef(); diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index 2351fb6cae351..802252bb63674 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -35,7 +34,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; -import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; @@ -47,9 +45,9 @@ public void testRectangleShape() throws IOException { int maxX = randomIntBetween(minX + 10, 180); int minY = randomIntBetween(-180, 170); int maxY = randomIntBetween(minY + 10, 180); - int[] x = new int[]{minX, maxX, maxX, minX, minX}; - int[] y = new int[]{minY, minY, maxY, maxY, minY}; - EdgeTreeWriter writer = new EdgeTreeWriter(x, y); + double[] x = new double[]{minX, maxX, maxX, minX, minX}; + double[] y = new double[]{minY, minY, maxY, maxY, minY}; + EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); @@ -113,15 +111,15 @@ public void testSimplePolygon() throws IOException { } builder = LegacyGeoShapeQueryProcessor.geometryToShapeBuilder(testPolygon); Rectangle box = builder.buildS4J().getBoundingBox(); - int minXBox = GeoEncodingUtils.encodeLongitude(box.getMinX()); - int minYBox = GeoEncodingUtils.encodeLatitude(box.getMinY()); - int maxXBox = GeoEncodingUtils.encodeLongitude(box.getMaxX()); - int maxYBox = GeoEncodingUtils.encodeLatitude(box.getMaxY()); + int minXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMinX()); + int minYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMinY()); + int maxXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMaxX()); + int maxYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMaxY()); - int[] x = asIntArray(testPolygon.getPolygon().getLons(), GeoEncodingUtils::encodeLongitude); - int[] y = asIntArray(testPolygon.getPolygon().getLats(), GeoEncodingUtils::encodeLatitude); + double[] x = testPolygon.getPolygon().getLons(); + double[] y = testPolygon.getPolygon().getLats(); - EdgeTreeWriter writer = new EdgeTreeWriter(x, y); + EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); @@ -147,8 +145,8 @@ public void testSimplePolygon() throws IOException { public void testPacMan() throws Exception { // pacman - int[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; - int[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; // candidate intersects cell int xMin = 2;//-5; @@ -157,7 +155,7 @@ public void testPacMan() throws Exception { int yMax = 1;//5; // test cell crossing poly - EdgeTreeWriter writer = new EdgeTreeWriter(px, py); + EdgeTreeWriter writer = new EdgeTreeWriter(px, py, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); @@ -166,17 +164,11 @@ public void testPacMan() throws Exception { } public void testGetShapeType() { - int[] pointCoord = new int[] { 0 }; - assertThat(new EdgeTreeWriter(pointCoord, pointCoord).getShapeType(), equalTo(ShapeType.LINESTRING)); - assertThat(new EdgeTreeWriter(List.of(pointCoord, pointCoord), List.of(pointCoord, pointCoord)).getShapeType(), + double[] pointCoord = new double[] { 0 }; + assertThat(new EdgeTreeWriter(pointCoord, pointCoord, TestCoordinateEncoder.INSTANCE).getShapeType(), + equalTo(ShapeType.LINESTRING)); + assertThat(new EdgeTreeWriter(List.of(pointCoord, pointCoord), List.of(pointCoord, pointCoord), + TestCoordinateEncoder.INSTANCE).getShapeType(), equalTo(ShapeType.MULTILINESTRING)); } - - private int[] asIntArray(double[] doub, Function encode) { - int[] intArr = new int[doub.length]; - for (int i = 0; i < intArr.length; i++) { - intArr[i] = encode.apply(doub[i]); - } - return intArr; - } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoderTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoderTests.java new file mode 100644 index 0000000000000..9a6e822a198da --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoShapeCoordinateEncoderTests.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.geo.GeoEncodingUtils; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.elasticsearch.common.geo.GeoShapeCoordinateEncoder.INSTANCE; + + +public class GeoShapeCoordinateEncoderTests extends ESTestCase { + + public void testLongitude() { + double randomLon = randomDoubleBetween(-180, 180, true); + double randomInvalidLon = randomFrom(randomDoubleBetween(-1000, -180.01, true), + randomDoubleBetween(180.01, 1000, true)); + + assertThat(INSTANCE.encodeX(Double.POSITIVE_INFINITY), equalTo(Integer.MAX_VALUE)); + assertThat(INSTANCE.encodeX(Double.NEGATIVE_INFINITY), equalTo(Integer.MIN_VALUE)); + int encodedLon = INSTANCE.encodeX(randomLon); + assertThat(encodedLon, equalTo(GeoEncodingUtils.encodeLongitude(randomLon))); + Exception e = expectThrows(IllegalArgumentException.class, () -> GeoShapeCoordinateEncoder.INSTANCE.encodeX(randomInvalidLon)); + assertThat(e.getMessage(), endsWith("must be between -180.0 and 180.0")); + + assertThat(INSTANCE.decodeX(encodedLon), closeTo(randomLon, 0.0001)); + assertThat(INSTANCE.decodeX(Integer.MAX_VALUE), closeTo(180, 0.00001)); + assertThat(INSTANCE.decodeX(Integer.MIN_VALUE), closeTo(-180, 0.00001)); + } + + public void testLatitude() { + double randomLat = randomDoubleBetween(-90, 90, true); + double randomInvalidLat = randomFrom(randomDoubleBetween(-1000, -90.01, true), + randomDoubleBetween(90.01, 1000, true)); + + assertThat(INSTANCE.encodeY(Double.POSITIVE_INFINITY), equalTo(Integer.MAX_VALUE)); + assertThat(INSTANCE.encodeY(Double.NEGATIVE_INFINITY), equalTo(Integer.MIN_VALUE)); + int encodedLat = INSTANCE.encodeY(randomLat); + assertThat(encodedLat, equalTo(GeoEncodingUtils.encodeLatitude(randomLat))); + Exception e = expectThrows(IllegalArgumentException.class, () -> GeoShapeCoordinateEncoder.INSTANCE.encodeY(randomInvalidLat)); + assertThat(e.getMessage(), endsWith("must be between -90.0 and 90.0")); + + assertThat(INSTANCE.decodeY(encodedLat), closeTo(randomLat, 0.0001)); + assertThat(INSTANCE.decodeY(Integer.MAX_VALUE), closeTo(90, 0.00001)); + assertThat(INSTANCE.decodeY(Integer.MIN_VALUE), closeTo(-90, 0.00001)); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 988bd976ce689..0387848f09a81 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.geo.GeoEncodingUtils; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Line; @@ -38,11 +37,6 @@ public class GeometryTreeTests extends ESTestCase { - static Extent geoExtent(double minX, double minY, double maxX, double maxY) { - return Extent.fromPoints(GeoEncodingUtils.encodeLongitude(minX), GeoEncodingUtils.encodeLatitude(minY), - GeoEncodingUtils.encodeLongitude(maxX), GeoEncodingUtils.encodeLatitude(maxY)); - } - public void testRectangleShape() throws IOException { for (int i = 0; i < 1000; i++) { int minX = randomIntBetween(-80, 70); @@ -53,50 +47,50 @@ public void testRectangleShape() throws IOException { double[] y = new double[]{minY, minY, maxY, maxY, minY}; Geometry rectangle = randomBoolean() ? new Polygon(new LinearRing(x, y), Collections.emptyList()) : new Rectangle(minX, maxX, maxY, minY); - GeometryTreeWriter writer = new GeometryTreeWriter(rectangle); + GeometryTreeWriter writer = new GeometryTreeWriter(rectangle, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertThat(geoExtent(minX, minY, maxX, maxY), equalTo(reader.getExtent())); + assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); // box-query touches bottom-left corner - assertTrue(reader.intersects(geoExtent(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), minX, minY))); + assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), minX, minY))); // box-query touches bottom-right corner - assertTrue(reader.intersects(geoExtent(maxX, minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), minY))); + assertTrue(reader.intersects(Extent.fromPoints(maxX, minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), minY))); // box-query touches top-right corner - assertTrue(reader.intersects(geoExtent(maxX, maxY, maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); + assertTrue(reader.intersects(Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); // box-query touches top-left corner - assertTrue(reader.intersects(geoExtent(minX - randomIntBetween(1, 10), maxY, minX, maxY + randomIntBetween(1, 10)))); + assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), maxY, minX, maxY + randomIntBetween(1, 10)))); // box-query fully-enclosed inside rectangle - assertTrue(reader.intersects(geoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, (3 * maxY + minY) / 4))); // box-query fully-contains poly - assertTrue(reader.intersects(geoExtent(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), + assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); // box-query half-in-half-out-right - assertTrue(reader.intersects(geoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), + assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), (3 * maxY + minY) / 4))); // box-query half-in-half-out-left - assertTrue(reader.intersects(geoExtent(minX - randomIntBetween(1, 10), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, + assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, (3 * maxY + minY) / 4))); // box-query half-in-half-out-top - assertTrue(reader.intersects(geoExtent((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), + assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); // box-query half-in-half-out-bottom - assertTrue(reader.intersects(geoExtent((3 * minX + maxX) / 4, minY - randomIntBetween(1, 10), + assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), (3 * maxY + minY) / 4))); // box-query outside to the right - assertFalse(reader.intersects(geoExtent(maxX + randomIntBetween(1, 4), minY, maxX + randomIntBetween(5, 10), maxY))); + assertFalse(reader.intersects(Extent.fromPoints(maxX + randomIntBetween(1, 4), minY, maxX + randomIntBetween(5, 10), maxY))); // box-query outside to the left - assertFalse(reader.intersects(geoExtent(maxX - randomIntBetween(5, 10), minY, minX - randomIntBetween(1, 4), maxY))); + assertFalse(reader.intersects(Extent.fromPoints(maxX - randomIntBetween(5, 10), minY, minX - randomIntBetween(1, 4), maxY))); // box-query outside to the top - assertFalse(reader.intersects(geoExtent(minX, maxY + randomIntBetween(1, 4), maxX, maxY + randomIntBetween(5, 10)))); + assertFalse(reader.intersects(Extent.fromPoints(minX, maxY + randomIntBetween(1, 4), maxX, maxY + randomIntBetween(5, 10)))); // box-query outside to the bottom - assertFalse(reader.intersects(geoExtent(minX, minY - randomIntBetween(5, 10), maxX, minY - randomIntBetween(1, 4)))); + assertFalse(reader.intersects(Extent.fromPoints(minX, minY - randomIntBetween(5, 10), maxX, minY - randomIntBetween(1, 4)))); } } @@ -106,15 +100,16 @@ public void testPacManPolygon() throws Exception { double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(py, px), Collections.emptyList())); + GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(py, px), Collections.emptyList()), + TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(geoExtent(2, -1, 11, 1))); - assertTrue(reader.intersects(geoExtent(-12, -12, 12, 12))); - assertTrue(reader.intersects(geoExtent(-2, -1, 2, 0))); - assertTrue(reader.intersects(geoExtent(-5, -6, 2, -2))); + assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); + assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); + assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); + assertTrue(reader.intersects(Extent.fromPoints(-5, -6, 2, -2))); } // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon @@ -122,18 +117,18 @@ public void testPolygonWithHole() throws Exception { Polygon polyWithHole = new Polygon(new LinearRing(new double[] { -50, 50, 50, -50, -50 }, new double[] { -50, -50, 50, 50, -50 }), Collections.singletonList(new LinearRing(new double[] { -10, 10, 10, -10, -10 }, new double[] { -10, -10, 10, 10, -10 }))); - GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole); + GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertFalse(reader.intersects(geoExtent(6, -6, 6, -6))); // in the hole - assertTrue(reader.intersects(geoExtent(25, -25, 25, -25))); // on the mainland - assertFalse(reader.intersects(geoExtent(51, 51, 52, 52))); // outside of mainland - assertTrue(reader.intersects(geoExtent(-60, -60, 60, 60))); // enclosing us completely - assertTrue(reader.intersects(geoExtent(49, 49, 51, 51))); // overlapping the mainland - assertTrue(reader.intersects(geoExtent(9, 9, 11, 11))); // overlapping the hole + assertFalse(reader.intersects(Extent.fromPoints(6, -6, 6, -6))); // in the hole + assertTrue(reader.intersects(Extent.fromPoints(25, -25, 25, -25))); // on the mainland + assertFalse(reader.intersects(Extent.fromPoints(51, 51, 52, 52))); // outside of mainland + assertTrue(reader.intersects(Extent.fromPoints(-60, -60, 60, 60))); // enclosing us completely + assertTrue(reader.intersects(Extent.fromPoints(49, 49, 51, 51))); // overlapping the mainland + assertTrue(reader.intersects(Extent.fromPoints(9, 9, 11, 11))); // overlapping the hole } public void testCombPolygon() throws Exception { @@ -145,14 +140,14 @@ public void testCombPolygon() throws Exception { Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole); + GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(geoExtent(5, 10, 5, 10))); - assertFalse(reader.intersects(geoExtent(15, 10, 15, 10))); - assertFalse(reader.intersects(geoExtent(25, 10, 25, 10))); + assertTrue(reader.intersects(Extent.fromPoints(5, 10, 5, 10))); + assertFalse(reader.intersects(Extent.fromPoints(15, 10, 15, 10))); + assertFalse(reader.intersects(Extent.fromPoints(25, 10, 25, 10))); } public void testPacManClosedLineString() throws Exception { @@ -161,15 +156,15 @@ public void testPacManClosedLineString() throws Exception { double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(new Line(px, py)); + GeometryTreeWriter writer = new GeometryTreeWriter(new Line(px, py), TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(geoExtent(2, -1, 11, 1))); - assertTrue(reader.intersects(geoExtent(-12, -12, 12, 12))); - assertTrue(reader.intersects(geoExtent(-2, -1, 2, 0))); - assertFalse(reader.intersects(geoExtent(-5, -6, 2, -2))); + assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); + assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); + assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); + assertFalse(reader.intersects(Extent.fromPoints(-5, -6, 2, -2))); } public void testPacManLineString() throws Exception { @@ -178,15 +173,15 @@ public void testPacManLineString() throws Exception { double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(new Line(px, py)); + GeometryTreeWriter writer = new GeometryTreeWriter(new Line(px, py), TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(geoExtent(2, -1, 11, 1))); - assertTrue(reader.intersects(geoExtent(-12, -12, 12, 12))); - assertTrue(reader.intersects(geoExtent(-2, -1, 2, 0))); - assertFalse(reader.intersects(geoExtent(-5, -6, 2, -2))); + assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); + assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); + assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); + assertFalse(reader.intersects(Extent.fromPoints(-5, -6, 2, -2))); } public void testPacManPoints() throws Exception { @@ -212,11 +207,11 @@ public void testPacManPoints() throws Exception { int yMax = 9; // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(new MultiPoint(points)); + GeometryTreeWriter writer = new GeometryTreeWriter(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); - assertTrue(reader.intersects(geoExtent(xMin, yMin, xMax, yMax))); + assertTrue(reader.intersects(Extent.fromPoints(xMin, yMin, xMax, yMax))); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java index 6e8a3d620a67a..ca035b03d4587 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java @@ -32,7 +32,7 @@ public class Point2DTests extends ESTestCase { public void testOnePoint() throws IOException { int x = randomIntBetween(-90, 90); int y = randomIntBetween(-90, 90); - Point2DWriter writer = new Point2DWriter(x, y); + Point2DWriter writer = new Point2DWriter(x, y, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); @@ -58,13 +58,13 @@ public void testPoints() throws IOException { Extent extent = Extent.fromPoints(minX, minY, maxX, maxY); int numPoints = randomIntBetween(2, 1000); - int[] x = new int[numPoints]; - int[] y = new int[numPoints]; + double[] x = new double[numPoints]; + double[] y = new double[numPoints]; for (int j = 0; j < numPoints; j++) { x[j] = randomIntBetween(minX, maxX); y[j] = randomIntBetween(minY, maxY); } - Point2DWriter writer = new Point2DWriter(x, y); + Point2DWriter writer = new Point2DWriter(x, y, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); diff --git a/server/src/test/java/org/elasticsearch/common/geo/TestCoordinateEncoder.java b/server/src/test/java/org/elasticsearch/common/geo/TestCoordinateEncoder.java new file mode 100644 index 0000000000000..222bfeff4e526 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/TestCoordinateEncoder.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + + +/** + * {@link CoordinateEncoder} used for tests that is an identity-encoder-decoder + */ +public class TestCoordinateEncoder implements CoordinateEncoder { + + public static final TestCoordinateEncoder INSTANCE = new TestCoordinateEncoder(); + + @Override + public int encodeX(double x) { + return (int) x; + } + + @Override + public int encodeY(double y) { + return (int) y; + } + + @Override + public double decodeX(int x) { + return x; + } + + @Override + public double decodeY(int y) { + return y; + } +} From 741a456a338d6d53f73bf947c48efb0414c5f017 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 10 Sep 2019 12:19:21 -0700 Subject: [PATCH 21/62] Add geo_shape support to geo_centroid aggregation (#46299) * Add geo_shape support to geo_centroid aggregation This commit annotates geometries with centroid information at index-time. Then each shapes centroid is used as input to the geo_centroid aggregator so that it effectively calculates the centroid of centroids of shapes within each bucket. The centroid of each geo_shape field is calculated as the centroid of all the points of all the shapes in the geometry collection. The centroid of each shape is calculated the same way geo_centroid was -- using a kahan-summation to reduce precision loss. * add centroid test --- .../common/geo/CentroidCalculator.java | 91 ++++++++++++ .../common/geo/EdgeTreeWriter.java | 21 ++- .../common/geo/GeometryTreeReader.java | 19 ++- .../common/geo/GeometryTreeWriter.java | 13 +- .../common/geo/Point2DWriter.java | 12 ++ .../common/geo/PolygonTreeWriter.java | 9 +- .../common/geo/ShapeTreeWriter.java | 2 + .../index/fielddata/MultiGeoValues.java | 18 ++- .../plain/LatLonShapeDVAtomicFieldData.java | 3 +- .../GeoCentroidAggregationBuilder.java | 6 +- .../metrics/GeoCentroidAggregator.java | 7 +- .../metrics/GeoCentroidAggregatorFactory.java | 6 +- .../common/geo/CentroidCalculatorTests.java | 44 ++++++ .../common/geo/EdgeTreeTests.java | 16 ++- .../common/geo/GeometryTreeTests.java | 17 ++- .../metrics/AbstractGeoTestCase.java | 3 +- .../metrics/GeoCentroidAggregatorTests.java | 133 +++++++++++++++++- .../aggregations/metrics/GeoCentroidIT.java | 18 ++- 18 files changed, 395 insertions(+), 43 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java new file mode 100644 index 0000000000000..c15c24d65ec40 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java @@ -0,0 +1,91 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +/** + * This class keeps a running Kahan-sum of coordinates + * that are to be averaged in {@link GeometryTreeWriter} for use + * as the centroid of a shape. + */ +public class CentroidCalculator { + + private double compX; + private double compY; + private double sumX; + private double sumY; + private int count; + + public CentroidCalculator() { + this.sumX = 0.0; + this.compX = 0.0; + this.sumY = 0.0; + this.compY = 0.0; + this.count = 0; + } + + /** + * adds a single coordinate to the running sum and count of coordinates + * for centroid calculation + * + * @param x the x-coordinate of the point + * @param y the y-coordinate of the point + */ + public void addCoordinate(double x, double y) { + double correctedX = x - compX; + double newSumX = sumX + correctedX; + compX = (newSumX - sumX) - correctedX; + sumX = newSumX; + + double correctedY = y - compY; + double newSumY = sumY + correctedY; + compY = (newSumY - sumY) - correctedY; + sumY = newSumY; + + count += 1; + } + + /** + * Adjusts the existing calculator to add the running sum and count + * from another {@link CentroidCalculator}. This is used to keep + * a running count of points from different sub-shapes of a single + * geo-shape field + * + * @param otherCalculator the other centroid calculator to add from + */ + void addFrom(CentroidCalculator otherCalculator) { + addCoordinate(otherCalculator.sumX, otherCalculator.sumY); + // adjust count + count += otherCalculator.count - 1; + } + + /** + * @return the x-coordinate centroid + */ + public double getX() { + return sumX / count; + } + + /** + * @return the y-coordinate centroid + */ + public double getY() { + return sumY / count; + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index 2fb2f1ef230fe..3b11d00833cfc 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -39,6 +39,7 @@ public class EdgeTreeWriter extends ShapeTreeWriter { private final Extent extent; private final int numShapes; + private final CentroidCalculator centroidCalculator; final Edge tree; @@ -46,12 +47,14 @@ public class EdgeTreeWriter extends ShapeTreeWriter { * @param x array of the x-coordinate of points. * @param y array of the y-coordinate of points. * @param coordinateEncoder class that encodes from real-valued x/y to serialized integer coordinate values. + * @param hasArea whether the tree represents a Polygon that has a defined area */ - EdgeTreeWriter(double[] x, double[] y, CoordinateEncoder coordinateEncoder) { - this(Collections.singletonList(x), Collections.singletonList(y), coordinateEncoder); + EdgeTreeWriter(double[] x, double[] y, CoordinateEncoder coordinateEncoder, boolean hasArea) { + this(Collections.singletonList(x), Collections.singletonList(y), coordinateEncoder, hasArea); } - EdgeTreeWriter(List x, List y, CoordinateEncoder coordinateEncoder) { + EdgeTreeWriter(List x, List y, CoordinateEncoder coordinateEncoder, boolean hasArea) { + this.centroidCalculator = new CentroidCalculator(); this.numShapes = x.size(); double top = Double.NEGATIVE_INFINITY; double bottom = Double.POSITIVE_INFINITY; @@ -59,6 +62,7 @@ public class EdgeTreeWriter extends ShapeTreeWriter { double negRight = Double.NEGATIVE_INFINITY; double posLeft = Double.POSITIVE_INFINITY; double posRight = Double.NEGATIVE_INFINITY; + List edges = new ArrayList<>(); for (int i = 0; i < y.size(); i++) { for (int j = 1; j < y.get(i).length; j++) { @@ -108,6 +112,12 @@ public class EdgeTreeWriter extends ShapeTreeWriter { if (x2 < 0 && x2 > negRight) { negRight = x2; } + + // calculate centroid + centroidCalculator.addCoordinate(x1, y1); + if (j == y.get(i).length - 1 && hasArea == false) { + centroidCalculator.addCoordinate(x2, y2); + } } } edges.sort(Edge::compareTo); @@ -127,6 +137,11 @@ public ShapeType getShapeType() { return numShapes > 1 ? ShapeType.MULTILINESTRING: ShapeType.LINESTRING; } + @Override + public CentroidCalculator getCentroidCalculator() { + return centroidCalculator; + } + @Override public void writeTo(StreamOutput out) throws IOException { extent.writeTo(out); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 9f442d29cc41e..364ee63ce1ba4 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -34,14 +34,17 @@ */ public class GeometryTreeReader { + private final int extentOffset = 8; private final ByteBufferStreamInput input; + private final CoordinateEncoder coordinateEncoder; - public GeometryTreeReader(BytesRef bytesRef) { + public GeometryTreeReader(BytesRef bytesRef, CoordinateEncoder coordinateEncoder) { this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + this.coordinateEncoder = coordinateEncoder; } public Extent getExtent() throws IOException { - input.position(0); + input.position(extentOffset); Extent extent = input.readOptionalWriteable(Extent::new); if (extent != null) { return extent; @@ -52,8 +55,18 @@ public Extent getExtent() throws IOException { return reader.getExtent(); } - public boolean intersects(Extent extent) throws IOException { + public double getCentroidX() throws IOException { input.position(0); + return coordinateEncoder.decodeX(input.readInt()); + } + + public double getCentroidY() throws IOException { + input.position(4); + return coordinateEncoder.decodeY(input.readInt()); + } + + public boolean intersects(Extent extent) throws IOException { + input.position(extentOffset); boolean hasExtent = input.readBoolean(); if (hasExtent) { Optional extentCheck = EdgeTreeReader.checkExtent(new Extent(input), extent); diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index a6c1dfb3a6300..61bda1d8c44ed 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -46,8 +46,12 @@ public class GeometryTreeWriter implements Writeable { private final GeometryTreeBuilder builder; + private final CoordinateEncoder coordinateEncoder; + private CentroidCalculator centroidCalculator; public GeometryTreeWriter(Geometry geometry, CoordinateEncoder coordinateEncoder) { + this.coordinateEncoder = coordinateEncoder; + this.centroidCalculator = new CentroidCalculator(); builder = new GeometryTreeBuilder(coordinateEncoder); geometry.visit(builder); } @@ -62,6 +66,8 @@ public void writeTo(StreamOutput out) throws IOException { // contains multiple sub-shapes boolean prependExtent = builder.shapeWriters.size() > 1; Extent extent = null; + out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); + out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); if (prependExtent) { extent = new Extent(builder.top, builder.bottom, builder.negLeft, builder.negRight, builder.posLeft, builder.posRight); } @@ -99,6 +105,7 @@ private void addWriter(ShapeTreeWriter writer) { posLeft = Math.min(posLeft, extent.posLeft); posRight = Math.max(posRight, extent.posRight); shapeWriters.add(writer); + centroidCalculator.addFrom(writer.getCentroidCalculator()); } @Override @@ -111,7 +118,7 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - addWriter(new EdgeTreeWriter(line.getLons(), line.getLats(), coordinateEncoder)); + addWriter(new EdgeTreeWriter(line.getLons(), line.getLats(), coordinateEncoder, false)); return null; } @@ -124,7 +131,7 @@ public Void visit(MultiLine multiLine) { x.add(line.getLons()); y.add(line.getLats()); } - addWriter(new EdgeTreeWriter(x, y, coordinateEncoder)); + addWriter(new EdgeTreeWriter(x, y, coordinateEncoder, false)); return null; } @@ -181,7 +188,7 @@ public Void visit(MultiPoint multiPoint) { @Override public Void visit(LinearRing ring) { - throw new IllegalArgumentException("invalid shape type found [Circle]"); + throw new IllegalArgumentException("invalid shape type found [LinearRing]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java index f59a806f958ab..bd2a250925b78 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java @@ -36,10 +36,12 @@ public class Point2DWriter extends ShapeTreeWriter { // size of a leaf node where searches are done sequentially. static final int LEAF_SIZE = 64; private final CoordinateEncoder coordinateEncoder; + private final CentroidCalculator centroidCalculator; Point2DWriter(double[] x, double[] y, CoordinateEncoder coordinateEncoder) { assert x.length == y.length; this.coordinateEncoder = coordinateEncoder; + this.centroidCalculator = new CentroidCalculator(); double top = Double.NEGATIVE_INFINITY; double bottom = Double.POSITIVE_INFINITY; double negLeft = Double.POSITIVE_INFINITY; @@ -47,6 +49,7 @@ public class Point2DWriter extends ShapeTreeWriter { double posLeft = Double.POSITIVE_INFINITY; double posRight = Double.NEGATIVE_INFINITY; coords = new double[x.length * K]; + for (int i = 0; i < x.length; i++) { double xi = x[i]; double yi = y[i]; @@ -66,6 +69,8 @@ public class Point2DWriter extends ShapeTreeWriter { } coords[2 * i] = xi; coords[2 * i + 1] = yi; + + centroidCalculator.addCoordinate(xi, yi); } sort(0, x.length - 1, 0); this.extent = new Extent(coordinateEncoder.encodeY(top), coordinateEncoder.encodeY(bottom), coordinateEncoder.encodeX(negLeft), @@ -76,6 +81,8 @@ public class Point2DWriter extends ShapeTreeWriter { this.coordinateEncoder = coordinateEncoder; coords = new double[] {x, y}; this.extent = Extent.fromPoint(coordinateEncoder.encodeX(x), coordinateEncoder.encodeY(y)); + this.centroidCalculator = new CentroidCalculator(); + centroidCalculator.addCoordinate(x, y); } @Override @@ -88,6 +95,11 @@ public ShapeType getShapeType() { return ShapeType.MULTIPOINT; } + @Override + public CentroidCalculator getCentroidCalculator() { + return centroidCalculator; + } + @Override public void writeTo(StreamOutput out) throws IOException { int numPoints = coords.length >> 1; diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java index b48ce0e1cb31b..d114ae40f84c7 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java @@ -35,8 +35,8 @@ public class PolygonTreeWriter extends ShapeTreeWriter { private final EdgeTreeWriter holes; public PolygonTreeWriter(double[] x, double[] y, List holesX, List holesY, CoordinateEncoder coordinateEncoder) { - outerShell = new EdgeTreeWriter(x, y, coordinateEncoder); - holes = holesX.isEmpty() ? null : new EdgeTreeWriter(holesX, holesY, coordinateEncoder); + outerShell = new EdgeTreeWriter(x, y, coordinateEncoder, true); + holes = holesX.isEmpty() ? null : new EdgeTreeWriter(holesX, holesY, coordinateEncoder, true); } public Extent getExtent() { @@ -47,6 +47,11 @@ public ShapeType getShapeType() { return ShapeType.POLYGON; } + @Override + public CentroidCalculator getCentroidCalculator() { + return outerShell.getCentroidCalculator(); + } + @Override public void writeTo(StreamOutput out) throws IOException { // calculate size of outerShell's tree to make it easy to jump to the holes tree quickly when querying diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java index 11f555b7176d5..35eaec5fb02f0 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java @@ -29,4 +29,6 @@ public abstract class ShapeTreeWriter implements Writeable { public abstract Extent getExtent(); public abstract ShapeType getShapeType(); + + public abstract CentroidCalculator getCentroidCalculator(); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 6486c74c7fb2a..c8dc4f7cb44b5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -128,14 +128,28 @@ public BoundingBox boundingBox() { return new BoundingBox(extent, GeoShapeCoordinateEncoder.INSTANCE); } + /** + * @return the latitude of the centroid of the shape + */ @Override public double lat() { - throw new UnsupportedOperationException("centroid of GeoShape is not defined"); + try { + return reader.getCentroidY(); + } catch (IOException e) { + throw new IllegalStateException("unable to read centroid of shape", e); + } } + /** + * @return the longitude of the centroid of the shape + */ @Override public double lon() { - throw new UnsupportedOperationException("centroid of GeoShape is not defined"); + try { + return reader.getCentroidX(); + } catch (IOException e) { + throw new IllegalStateException("unable to read centroid of shape", e); + } } public static GeoShapeValue missing(String missing) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java index ec03a3959d482..2e0480263174c 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.GeometryTreeReader; import org.elasticsearch.index.fielddata.MultiGeoValues; @@ -74,7 +75,7 @@ public int docValueCount() { @Override public GeoValue nextValue() throws IOException { final BytesRef encoded = binaryValues.binaryValue(); - return new GeoShapeValue(new GeometryTreeReader(encoded)); + return new GeoShapeValue(new GeometryTreeReader(encoded, GeoShapeCoordinateEncoder.INSTANCE)); } }; } catch (IOException e) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java index af2d1a600243a..6f3d360abb0f8 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregationBuilder.java @@ -39,13 +39,13 @@ import java.util.Map; public class GeoCentroidAggregationBuilder - extends ValuesSourceAggregationBuilder.LeafOnly { + extends ValuesSourceAggregationBuilder.LeafOnly { public static final String NAME = "geo_centroid"; private static final ObjectParser PARSER; static { PARSER = new ObjectParser<>(GeoCentroidAggregationBuilder.NAME); - ValuesSourceParserHelper.declareGeoPointFields(PARSER, true, false); + ValuesSourceParserHelper.declareGeoFields(PARSER, true, false); } public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException { @@ -78,7 +78,7 @@ protected void innerWriteTo(StreamOutput out) { } @Override - protected GeoCentroidAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig config, + protected GeoCentroidAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { return new GeoCentroidAggregatorFactory(name, config, context, parent, subFactoriesBuilder, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java index 975356c7d6fe3..c91ebb92e8714 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java @@ -42,12 +42,12 @@ * A geo metric aggregator that computes a geo-centroid from a {@code geo_point} type field */ final class GeoCentroidAggregator extends MetricsAggregator { - private final ValuesSource.GeoPoint valuesSource; + private final ValuesSource.Geo valuesSource; private DoubleArray lonSum, lonCompensations, latSum, latCompensations; private LongArray counts; GeoCentroidAggregator(String name, SearchContext context, Aggregator parent, - ValuesSource.GeoPoint valuesSource, List pipelineAggregators, + ValuesSource.Geo valuesSource, List pipelineAggregators, Map metaData) throws IOException { super(name, context, parent, pipelineAggregators, metaData); this.valuesSource = valuesSource; @@ -89,6 +89,9 @@ public void collect(int doc, long bucket) throws IOException { double compensationLon = lonCompensations.get(bucket); // update the sum + // + // this calculates the centroid of centroid of shapes when + // executing against geo-shape fields. for (int i = 0; i < valueCount; ++i) { MultiGeoValues.GeoValue value = values.nextValue(); //latitude diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java index b12ce921b7d52..3dd694fc21b7f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorFactory.java @@ -32,9 +32,9 @@ import java.util.List; import java.util.Map; -class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory { +class GeoCentroidAggregatorFactory extends ValuesSourceAggregatorFactory { - GeoCentroidAggregatorFactory(String name, ValuesSourceConfig config, + GeoCentroidAggregatorFactory(String name, ValuesSourceConfig config, SearchContext context, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { super(name, config, context, parent, subFactoriesBuilder, metaData); @@ -47,7 +47,7 @@ protected Aggregator createUnmapped(Aggregator parent, } @Override - protected Aggregator doCreateInternal(ValuesSource.GeoPoint valuesSource, Aggregator parent, + protected Aggregator doCreateInternal(ValuesSource.Geo valuesSource, Aggregator parent, boolean collectsFromSingleBucket, List pipelineAggregators, Map metaData) throws IOException { return new GeoCentroidAggregator(name, context, parent, valuesSource, pipelineAggregators, metaData); diff --git a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java new file mode 100644 index 0000000000000..43c5e4a02134b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class CentroidCalculatorTests extends ESTestCase { + + public void test() { + double[] x = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + double[] y = new double[] { 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 }; + double[] xRunningAvg = new double[] { 1, 1.5, 2.0, 2.5, 3, 3.5, 4, 4.5, 5, 5.5 }; + double[] yRunningAvg = new double[] { 10, 15, 20, 25, 30, 35, 40, 45, 50, 55 }; + CentroidCalculator calculator = new CentroidCalculator(); + for (int i = 0; i < 10; i++) { + calculator.addCoordinate(x[i], y[i]); + assertThat(calculator.getX(), equalTo(xRunningAvg[i])); + assertThat(calculator.getY(), equalTo(yRunningAvg[i])); + } + CentroidCalculator otherCalculator = new CentroidCalculator(); + otherCalculator.addCoordinate(0.0, 0.0); + calculator.addFrom(otherCalculator); + assertThat(calculator.getX(), equalTo(5.0)); + assertThat(calculator.getY(), equalTo(50.0)); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index 802252bb63674..4bdeba2e55612 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -47,11 +47,15 @@ public void testRectangleShape() throws IOException { int maxY = randomIntBetween(minY + 10, 180); double[] x = new double[]{minX, maxX, maxX, minX, minX}; double[] y = new double[]{minY, minY, maxY, maxY, minY}; - EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE); + EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE, true); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); + EdgeTreeReader reader = new EdgeTreeReader( + new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); + + assertThat(writer.getCentroidCalculator().getX(), equalTo((minX + maxX)/2.0)); + assertThat(writer.getCentroidCalculator().getY(), equalTo((minY + maxY)/2.0)); // box-query touches bottom-left corner assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY))); @@ -119,7 +123,7 @@ public void testSimplePolygon() throws IOException { double[] x = testPolygon.getPolygon().getLons(); double[] y = testPolygon.getPolygon().getLats(); - EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE); + EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE, true); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); @@ -155,7 +159,7 @@ public void testPacMan() throws Exception { int yMax = 1;//5; // test cell crossing poly - EdgeTreeWriter writer = new EdgeTreeWriter(px, py, TestCoordinateEncoder.INSTANCE); + EdgeTreeWriter writer = new EdgeTreeWriter(px, py, TestCoordinateEncoder.INSTANCE, true); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); @@ -165,10 +169,10 @@ public void testPacMan() throws Exception { public void testGetShapeType() { double[] pointCoord = new double[] { 0 }; - assertThat(new EdgeTreeWriter(pointCoord, pointCoord, TestCoordinateEncoder.INSTANCE).getShapeType(), + assertThat(new EdgeTreeWriter(pointCoord, pointCoord, TestCoordinateEncoder.INSTANCE, false).getShapeType(), equalTo(ShapeType.LINESTRING)); assertThat(new EdgeTreeWriter(List.of(pointCoord, pointCoord), List.of(pointCoord, pointCoord), - TestCoordinateEncoder.INSTANCE).getShapeType(), + TestCoordinateEncoder.INSTANCE, false).getShapeType(), equalTo(ShapeType.MULTILINESTRING)); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 0387848f09a81..d8abe1aaadd69 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -52,9 +52,12 @@ public void testRectangleShape() throws IOException { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); + // encoder loses precision when casting to integer, so centroid is calculated using integer division here + assertThat(reader.getCentroidX(), equalTo((double) ((minX + maxX)/2))); + assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY)/2))); // box-query touches bottom-left corner assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), minX, minY))); @@ -105,7 +108,7 @@ public void testPacManPolygon() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); @@ -121,7 +124,7 @@ public void testPolygonWithHole() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), null); assertFalse(reader.intersects(Extent.fromPoints(6, -6, 6, -6))); // in the hole assertTrue(reader.intersects(Extent.fromPoints(25, -25, 25, -25))); // on the mainland @@ -144,7 +147,7 @@ public void testCombPolygon() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(5, 10, 5, 10))); assertFalse(reader.intersects(Extent.fromPoints(15, 10, 15, 10))); assertFalse(reader.intersects(Extent.fromPoints(25, 10, 25, 10))); @@ -160,7 +163,7 @@ public void testPacManClosedLineString() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); @@ -177,7 +180,7 @@ public void testPacManLineString() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); @@ -211,7 +214,7 @@ public void testPacManPoints() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef()); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(xMin, yMin, xMax, yMax))); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java index 5d6c10079d145..3883eeeab9c49 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -68,7 +68,7 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase { protected static int numUniqueGeoPoints; protected static GeoPoint[] singleValues, multiValues; protected static GeoPoint singleTopLeft, singleBottomRight, multiTopLeft, multiBottomRight, - singleCentroid, multiCentroid, unmappedCentroid; + singleCentroid, singleShapeCentroid, multiCentroid, unmappedCentroid; protected static ObjectIntMap expectedDocCountsForGeoHash = null; protected static ObjectObjectMap expectedCentroidsForGeoHash = null; protected static final double GEOHASH_TOLERANCE = 1E-5D; @@ -85,6 +85,7 @@ public void setupSuiteScopeCluster() throws Exception { multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); multiBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); singleCentroid = new GeoPoint(0, 0); + singleShapeCentroid = new GeoPoint(9.4, 34.4); multiCentroid = new GeoPoint(0, 0); unmappedCentroid = new GeoPoint(0, 0); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java index 4acae1764953a..83e3887e323f8 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java @@ -25,14 +25,32 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomGeoGenerator; import java.io.IOException; +import java.util.function.Function; public class GeoCentroidAggregatorTests extends AggregatorTestCase { @@ -84,7 +102,7 @@ public void testUnmapped() throws Exception { } } - public void testSingleValuedField() throws Exception { + public void testSingleValuedGeoPointField() throws Exception { int numDocs = scaledRandomIntBetween(64, 256); int numUniqueGeoPoints = randomIntBetween(1, numDocs); try (Directory dir = newDirectory(); @@ -103,11 +121,11 @@ public void testSingleValuedField() throws Exception { expectedCentroid = expectedCentroid.reset(expectedCentroid.lat() + (singleVal.lat() - expectedCentroid.lat()) / (i + 1), expectedCentroid.lon() + (singleVal.lon() - expectedCentroid.lon()) / (i + 1)); } - assertCentroid(w, expectedCentroid); + assertCentroid(w, expectedCentroid, new GeoPointFieldMapper.GeoPointFieldType()); } } - public void testMultiValuedField() throws Exception { + public void testMultiValuedGeoPointField() throws Exception { int numDocs = scaledRandomIntBetween(64, 256); int numUniqueGeoPoints = randomIntBetween(1, numDocs); try (Directory dir = newDirectory(); @@ -131,12 +149,115 @@ public void testMultiValuedField() throws Exception { expectedCentroid = expectedCentroid.reset(expectedCentroid.lat() + (newMVLat - expectedCentroid.lat()) / (i + 1), expectedCentroid.lon() + (newMVLon - expectedCentroid.lon()) / (i + 1)); } - assertCentroid(w, expectedCentroid); + assertCentroid(w, expectedCentroid, new GeoPointFieldMapper.GeoPointFieldType()); } } - private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) throws IOException { - MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType(); + @SuppressWarnings("unchecked") + public void testGeoShapeField() throws Exception { + int numDocs = scaledRandomIntBetween(64, 256); + Function geometryGenerator = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPoint, + (hasAlt) -> GeometryTestUtils.randomRectangle(), + GeometryTestUtils::randomMultiPolygon + ); + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + GeoPoint expectedCentroid = new GeoPoint(0, 0); + CentroidCalculator centroidOfCentroidsCalculator = new CentroidCalculator(); + for (int i = 0; i < numDocs; i++) { + CentroidCalculator calculator = new CentroidCalculator(); + Document document = new Document(); + Geometry geometry = geometryGenerator.apply(false); + geometry.visit(new GeometryVisitor() { + @Override + public Void visit(Circle circle) throws Exception { + calculator.addCoordinate(circle.getX(), circle.getY()); + return null; + } + + @Override + public Void visit(GeometryCollection collection) throws Exception { + for (Geometry shape : collection) { + shape.visit(this); + } + return null; + } + + @Override + public Void visit(Line line) throws Exception { + for (int i = 0; i < line.length(); i++) { + calculator.addCoordinate(line.getX(i), line.getY(i)); + } + return null; + } + + @Override + public Void visit(LinearRing ring) throws Exception { + for (int i = 0; i < ring.length() - 1; i++) { + calculator.addCoordinate(ring.getX(i), ring.getY(i)); + } + return null; + } + + @Override + public Void visit(MultiLine multiLine) throws Exception { + for (Line line : multiLine) { + visit(line); + } + return null; + } + + @Override + public Void visit(MultiPoint multiPoint) throws Exception { + for (Point point : multiPoint) { + visit(point); + } + return null; + } + + @Override + public Void visit(MultiPolygon multiPolygon) throws Exception { + for (Polygon polygon : multiPolygon) { + visit(polygon); + } + return null; + } + + @Override + public Void visit(Point point) throws Exception { + calculator.addCoordinate(point.getX(), point.getY()); + return null; + } + + @Override + public Void visit(Polygon polygon) throws Exception { + return visit(polygon.getPolygon()); + } + + @Override + public Void visit(Rectangle rectangle) throws Exception { + calculator.addCoordinate(rectangle.getMinX(), rectangle.getMinY()); + calculator.addCoordinate(rectangle.getMinX(), rectangle.getMaxY()); + calculator.addCoordinate(rectangle.getMaxX(), rectangle.getMinY()); + calculator.addCoordinate(rectangle.getMaxX(), rectangle.getMaxY()); + return null; + } + }); + document.add(new BinaryGeoShapeDocValuesField("field", geometry)); + w.addDocument(document); + centroidOfCentroidsCalculator.addCoordinate(calculator.getX(), calculator.getY()); + } + expectedCentroid.reset(centroidOfCentroidsCalculator.getY(), centroidOfCentroidsCalculator.getX()); + assertCentroid(w, expectedCentroid, new GeoShapeFieldMapper.GeoShapeFieldType()); + } + } + + private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid, MappedFieldType fieldType) throws IOException { fieldType.setHasDocValues(true); fieldType.setName("field"); GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg") diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java index d941ce27ede2f..576c4a34e347c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java @@ -106,6 +106,22 @@ public void testSingleValuedField() throws Exception { assertEquals(numDocs, geoCentroid.count()); } + public void testShapeField() throws Exception { + SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) + .setQuery(matchAllQuery()) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME)) + .get(); + assertSearchResponse(response); + + GeoCentroid geoCentroid = response.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid.lat(), closeTo(singleShapeCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(centroid.lon(), closeTo(singleShapeCentroid.lon(), GEOHASH_TOLERANCE)); + assertEquals(5, geoCentroid.count()); + } + public void testSingleValueFieldGetProperty() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) .setQuery(matchAllQuery()) @@ -123,7 +139,7 @@ public void testSingleValueFieldGetProperty() throws Exception { GeoCentroid geoCentroid = global.getAggregations().get(aggName); assertThat(geoCentroid, notNullValue()); assertThat(geoCentroid.getName(), equalTo(aggName)); - assertThat((GeoCentroid) ((InternalAggregation)global).getProperty(aggName), sameInstance(geoCentroid)); + assertThat(((InternalAggregation)global).getProperty(aggName), sameInstance(geoCentroid)); GeoPoint centroid = geoCentroid.centroid(); assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); From 0fd63b818f70f6291c5eee6df58687e89daddb2e Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 20 Sep 2019 07:45:48 -0700 Subject: [PATCH 22/62] geo-grid aggregation support for geo-shapes (#46513) adds support for geo_shape fields in geohash_grid and geotile_grid aggregations --- .../elasticsearch/geometry/utils/Geohash.java | 24 +++ .../org/elasticsearch/common/geo/Extent.java | 2 +- .../index/fielddata/FieldData.java | 6 + .../index/fielddata/MultiGeoValues.java | 55 ++++++- .../SingletonMultiGeoPointValues.java | 7 + .../plain/LatLonPointDVAtomicFieldData.java | 6 + .../plain/LatLonShapeDVAtomicFieldData.java | 6 + .../GeoTileGridValuesSourceBuilder.java | 12 +- .../bucket/geogrid/CellIdSource.java | 48 +++--- .../geogrid/GeoGridAggregationBuilder.java | 16 +- .../bucket/geogrid/GeoGridTiler.java | 144 ++++++++++++++++++ .../GeoHashGridAggregationBuilder.java | 4 +- .../geogrid/GeoHashGridAggregatorFactory.java | 12 +- .../GeoTileGridAggregationBuilder.java | 4 +- .../geogrid/GeoTileGridAggregatorFactory.java | 11 +- .../bucket/geogrid/GeoTileUtils.java | 68 ++++++++- .../aggregations/support/MissingValues.java | 5 + .../ScriptDocValuesGeoPointsTests.java | 6 + .../geogrid/GeoGridAggregatorTestCase.java | 75 ++++++++- .../bucket/geogrid/GeoGridTilerTests.java | 110 +++++++++++++ .../geogrid/GeoTileGridAggregatorTests.java | 1 - .../support/MissingValuesTests.java | 5 + 22 files changed, 567 insertions(+), 60 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java create mode 100644 server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java index f67c404dc4188..f0924905aea5d 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java @@ -53,6 +53,20 @@ public class Geohash { /** Bit encoded representation of the latitude of north pole */ private static final long MAX_LAT_BITS = (0x1L << (PRECISION * 5 / 2)) - 1; + // Below code is adapted from the spatial4j library (GeohashUtils.java) Apache 2.0 Licensed + private static final double[] precisionToLatHeight, precisionToLonWidth; + static { + precisionToLatHeight = new double[PRECISION + 1]; + precisionToLonWidth = new double[PRECISION + 1]; + precisionToLatHeight[0] = 90*2; + precisionToLonWidth[0] = 180*2; + boolean even = false; + for(int i = 1; i <= PRECISION; i++) { + precisionToLatHeight[i] = precisionToLatHeight[i-1] / (even ? 8 : 4); + precisionToLonWidth[i] = precisionToLonWidth[i-1] / (even ? 4 : 8); + even = ! even; + } + } // no instance: private Geohash() { @@ -316,6 +330,16 @@ public static int encodeLongitude(double longitude) { return (int) Math.floor(longitude / LON_DECODE); } + /** approximate width of geohash tile for a specific precision in degrees */ + public static double lonWidthInDegrees(int precision) { + return precisionToLonWidth[precision]; + } + + /** approximate height of geohash tile for a specific precision in degrees */ + public static double latHeightInDegrees(int precision) { + return precisionToLatHeight[precision]; + } + /** returns the latitude value from the string based geohash */ public static final double decodeLatitude(final String geohash) { return decodeLatitude(Geohash.mortonEncode(geohash)); diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index c7f9c69942893..46991c8a2801f 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -39,7 +39,7 @@ public class Extent implements Writeable { public final int posLeft; public final int posRight; - Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + public Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { this.top = top; this.bottom = bottom; this.negLeft = negLeft; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java index e2e3cade58558..d8b2309748379 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldData.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.ArrayList; @@ -83,6 +84,11 @@ public int docValueCount() { return 0; } + @Override + public ValuesSourceType valuesSourceType() { + return ValuesSourceType.GEO; + } + @Override public GeoValue nextValue() { throw new UnsupportedOperationException(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index c8dc4f7cb44b5..bb2290738d862 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -25,8 +25,10 @@ import org.elasticsearch.common.geo.GeometryTreeReader; import org.elasticsearch.common.geo.GeometryTreeWriter; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.GeographyValidator; import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.text.ParseException; @@ -65,6 +67,8 @@ protected MultiGeoValues() { */ public abstract int docValueCount(); + public abstract ValuesSourceType valuesSourceType(); + /** * Return the next value associated with the current document. This must not be * called more than {@link #docValueCount()} times. @@ -91,6 +95,11 @@ public BoundingBox boundingBox() { return new BoundingBox(geoPoint); } + @Override + public boolean intersects(Rectangle rectangle) { + throw new UnsupportedOperationException("intersect is unsupported for geo_point doc values"); + } + @Override public double lat() { return geoPoint.lat(); @@ -131,6 +140,20 @@ public BoundingBox boundingBox() { /** * @return the latitude of the centroid of the shape */ + @Override + public boolean intersects(Rectangle rectangle) { + int minX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMinX()); + int maxX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMaxX()); + int minY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMinY()); + int maxY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMaxY()); + Extent extent = new Extent(maxY, minY, minX, maxX, minX, maxX); + try { + return reader.intersects(extent); + } catch (IOException e) { + throw new IllegalStateException("unable to check intersection", e); + } + } + @Override public double lat() { try { @@ -171,6 +194,7 @@ public interface GeoValue { double lat(); double lon(); BoundingBox boundingBox(); + boolean intersects(Rectangle rectangle); } public static class BoundingBox { @@ -181,7 +205,7 @@ public static class BoundingBox { public final double posLeft; public final double posRight; - BoundingBox(Extent extent, CoordinateEncoder coordinateEncoder) { + public BoundingBox(Extent extent, CoordinateEncoder coordinateEncoder) { this.top = coordinateEncoder.decodeY(extent.top); this.bottom = coordinateEncoder.decodeY(extent.bottom); if (extent.negLeft == Integer.MAX_VALUE) { @@ -221,5 +245,34 @@ public static class BoundingBox { this.posRight = point.lon(); } } + /** + * @return the minimum y-coordinate of the extent + */ + public double minY() { + return bottom; + } + + /** + * @return the maximum y-coordinate of the extent + */ + public double maxY() { + return top; + } + + /** + * @return the absolute minimum x-coordinate of the extent, whether it is positive or negative. + */ + public double minX() { + return Math.min(negLeft, posLeft); + } + + /** + * @return the absolute maximum x-coordinate of the extent, whether it is positive or negative. + */ + public double maxX() { + return Math.max(negRight, posRight); + } + + } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java index 2e56a42e1604b..46123fa2fbc40 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SingletonMultiGeoPointValues.java @@ -19,6 +19,8 @@ package org.elasticsearch.index.fielddata; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; + import java.io.IOException; final class SingletonMultiGeoPointValues extends MultiGeoValues { @@ -39,6 +41,11 @@ public int docValueCount() { return 1; } + @Override + public ValuesSourceType valuesSourceType() { + return ValuesSourceType.GEOPOINT; + } + @Override public GeoValue nextValue() throws IOException { return in.nextValue(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java index b2afd1ad045b7..afa38bf785b43 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonPointDVAtomicFieldData.java @@ -25,6 +25,7 @@ import org.apache.lucene.util.Accountable; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Collection; @@ -73,6 +74,11 @@ public int docValueCount() { return numericValues.docValueCount(); } + @Override + public ValuesSourceType valuesSourceType() { + return ValuesSourceType.GEOPOINT; + } + @Override public GeoValue nextValue() throws IOException { final long encoded = numericValues.nextValue(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java index 2e0480263174c..ed04d496a2704 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java @@ -26,6 +26,7 @@ import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.GeometryTreeReader; import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.util.Collection; @@ -72,6 +73,11 @@ public int docValueCount() { return 1; } + @Override + public ValuesSourceType valuesSourceType() { + return ValuesSourceType.GEOSHAPE; + } + @Override public GeoValue nextValue() throws IOException { final BytesRef encoded = binaryValues.binaryValue(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index 17a5b3c0e9993..80bdf73f7d5de 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridTiler; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.search.aggregations.support.ValueType; @@ -106,16 +107,17 @@ public boolean equals(Object obj) { protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardContext, ValuesSourceConfig config) throws IOException { ValuesSource orig = config.toValuesSource(queryShardContext); if (orig == null) { - orig = ValuesSource.GeoPoint.EMPTY; + orig = ValuesSource.Geo.EMPTY; } - if (orig instanceof ValuesSource.GeoPoint) { - ValuesSource.GeoPoint geoPoint = (ValuesSource.GeoPoint) orig; + if (orig instanceof ValuesSource.Geo) { + ValuesSource.Geo geoValue = (ValuesSource.Geo) orig; // is specified in the builder. final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null; - CellIdSource cellIdSource = new CellIdSource(geoPoint, precision, GeoTileUtils::longEncode); + CellIdSource cellIdSource = new CellIdSource(geoValue, precision, GeoGridTiler.GeoTileGridTiler.INSTANCE); return new CompositeValuesSourceConfig(name, fieldType, cellIdSource, DocValueFormat.GEOTILE, order(), missingBucket()); } else { - throw new IllegalArgumentException("invalid source, expected geo_point, got " + orig.getClass().getSimpleName()); + throw new IllegalArgumentException("invalid source, expected one of [geo_point, geo_shape], got " + + orig.getClass().getSimpleName()); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java index 0de149c6b560f..6244feae4ee65 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java @@ -33,11 +33,11 @@ * to numeric long values for bucketing. */ public class CellIdSource extends ValuesSource.Numeric { - private final ValuesSource.GeoPoint valuesSource; + private final ValuesSource.Geo valuesSource; private final int precision; - private final GeoPointLongEncoder encoder; + private final GeoGridTiler encoder; - public CellIdSource(GeoPoint valuesSource, int precision, GeoPointLongEncoder encoder) { + public CellIdSource(Geo valuesSource, int precision, GeoGridTiler encoder) { this.valuesSource = valuesSource; //different GeoPoints could map to the same or different hashing cells. this.precision = precision; @@ -68,33 +68,43 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext ctx) { throw new UnsupportedOperationException(); } - /** - * The encoder to use to convert a geopoint's (lon, lat, precision) into - * a long-encoded bucket key for aggregating. - */ - @FunctionalInterface - public interface GeoPointLongEncoder { - long encode(double lon, double lat, int precision); - } - private static class CellValues extends AbstractSortingNumericDocValues { private MultiGeoValues geoValues; private int precision; - private GeoPointLongEncoder encoder; + private GeoGridTiler tiler; - protected CellValues(MultiGeoValues geoValues, int precision, GeoPointLongEncoder encoder) { + protected CellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { this.geoValues = geoValues; this.precision = precision; - this.encoder = encoder; + this.tiler = tiler; } @Override public boolean advanceExact(int docId) throws IOException { if (geoValues.advanceExact(docId)) { - resize(geoValues.docValueCount()); - for (int i = 0; i < docValueCount(); ++i) { - MultiGeoValues.GeoValue target = geoValues.nextValue(); - values[i] = encoder.encode(target.lon(), target.lat(), precision); + switch (geoValues.valuesSourceType()) { + case GEOPOINT: + resize(geoValues.docValueCount()); + for (int i = 0; i < docValueCount(); ++i) { + MultiGeoValues.GeoValue target = geoValues.nextValue(); + values[i] = tiler.encode(target.lon(), target.lat(), precision); + } + break; + case GEOSHAPE: + case GEO: + MultiGeoValues.GeoValue target = geoValues.nextValue(); + // TODO(talevy): determine reasonable circuit-breaker here + // must resize array to contain the upper-bound of matching cells, which + // is the number of tiles that overlap the shape's bounding-box. No need + // to be concerned with original docValueCount since shape doc-values are + // single-valued. + resize((int) tiler.getBoundingTileCount(target, precision)); + int matched = tiler.setValues(values, target, precision); + // must truncate array to only contain cells that actually intersected shape + resize(matched); + break; + default: + throw new IllegalArgumentException("unsupported geo type"); } sort(); return true; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index c83ce8582826d..6a50b33428dcf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -43,7 +43,7 @@ import java.util.Map; import java.util.Objects; -public abstract class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder +public abstract class GeoGridAggregationBuilder extends ValuesSourceAggregationBuilder implements MultiBucketAggregationBuilder { /* recognized field names in JSON */ static final ParseField FIELD_PRECISION = new ParseField("precision"); @@ -61,7 +61,7 @@ protected interface PrecisionParser { public static ObjectParser createParser(String name, PrecisionParser precisionParser) { ObjectParser parser = new ObjectParser<>(name); - ValuesSourceParserHelper.declareGeoPointFields(parser, false, false); + ValuesSourceParserHelper.declareGeoFields(parser, false, false); parser.declareField((p, builder, context) -> builder.precision(precisionParser.parse(p)), FIELD_PRECISION, org.elasticsearch.common.xcontent.ObjectParser.ValueType.INT); parser.declareInt(GeoGridAggregationBuilder::size, FIELD_SIZE); @@ -70,7 +70,7 @@ public static ObjectParser createParser(String } public GeoGridAggregationBuilder(String name) { - super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(name, ValuesSourceType.GEO, ValueType.GEO); } protected GeoGridAggregationBuilder(GeoGridAggregationBuilder clone, Builder factoriesBuilder, Map metaData) { @@ -85,7 +85,7 @@ protected GeoGridAggregationBuilder(GeoGridAggregationBuilder clone, Builder fac * Read from a stream. */ public GeoGridAggregationBuilder(StreamInput in) throws IOException { - super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT); + super(in, ValuesSourceType.GEO, ValueType.GEO); precision = in.readVInt(); requiredSize = in.readVInt(); shardSize = in.readVInt(); @@ -108,8 +108,8 @@ protected void innerWriteTo(StreamOutput out) throws IOException { /** * Creates a new instance of the {@link ValuesSourceAggregatorFactory}-derived class specific to the geo aggregation. */ - protected abstract ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + protected abstract ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, QueryShardContext queryShardContext, AggregatorFactory parent, Builder subFactoriesBuilder, Map metaData ) throws IOException; @@ -144,8 +144,8 @@ public int shardSize() { } @Override - protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, - ValuesSourceConfig config, + protected ValuesSourceAggregatorFactory innerBuild(QueryShardContext queryShardContext, + ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { int shardSize = this.shardSize; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java new file mode 100644 index 0000000000000..aef09b458630c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -0,0 +1,144 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +/** + * The tiler to use to convert a geo value into long-encoded bucket keys for aggregating. + */ +public interface GeoGridTiler { + /** + * encodes a single point to its long-encoded bucket key value. + * + * @param x the x-coordinate + * @param y the y-coordinate + * @param precision the zoom level of tiles + */ + long encode(double x, double y, int precision); + + /** + * computes the number of tiles for a specific precision that the geo value's + * bounding-box is contained within. + * + * @param geoValue the input shape + * @param precision the tile zoom-level + */ + long getBoundingTileCount(MultiGeoValues.GeoValue geoValue, int precision); + + /** + * + * @param docValues the array of long-encoded bucket keys to fill + * @param geoValue the input shape + * @param precision the tile zoom-level + * + * @return the number of tiles the geoValue intersects + */ + int setValues(long[] docValues, MultiGeoValues.GeoValue geoValue, int precision); + + class GeoHashGridTiler implements GeoGridTiler { + public static final GeoHashGridTiler INSTANCE = new GeoHashGridTiler(); + + private GeoHashGridTiler() {} + + @Override + public long encode(double x, double y, int precision) { + return Geohash.longEncode(x, y, precision); + } + + @Override + public long getBoundingTileCount(MultiGeoValues.GeoValue geoValue, int precision) { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + // find minimum (x,y) of geo-hash-cell that contains (bounds.minX, bounds.minY) + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + Rectangle geoHashCell = Geohash.toBoundingBox(hash); + long numLonCells = Math.max(1, (long) Math.ceil( + (bounds.maxX() - geoHashCell.getMinX()) / Geohash.lonWidthInDegrees(precision))); + long numLatCells = Math.max(1, (long) Math.ceil( + (bounds.maxY() - geoHashCell.getMinY()) / Geohash.latHeightInDegrees(precision))); + return numLonCells * numLatCells; + } + + @Override + public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precision) { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + int idx = 0; + // find minimum (x,y) of geo-hash-cell that contains (bounds.minX, bounds.minY) + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + Rectangle geoHashCell = Geohash.toBoundingBox(hash); + for (double i = geoHashCell.getMinX(); i < bounds.maxX(); i+= Geohash.lonWidthInDegrees(precision)) { + for (double j = geoHashCell.getMinY(); j < bounds.maxY(); j += Geohash.latHeightInDegrees(precision)) { + Rectangle rectangle = Geohash.toBoundingBox(Geohash.stringEncode(i, j, precision)); + if (geoValue.intersects(rectangle)) { + values[idx++] = encode(i, j, precision); + } + } + } + + return idx; + } + } + + class GeoTileGridTiler implements GeoGridTiler { + public static final GeoTileGridTiler INSTANCE = new GeoTileGridTiler(); + + private GeoTileGridTiler() {} + + @Override + public long encode(double x, double y, int precision) { + return GeoTileUtils.longEncode(x, y, precision); + } + + @Override + public long getBoundingTileCount(MultiGeoValues.GeoValue geoValue, int precision) { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + final double tiles = 1 << precision; + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + return (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1); + } + + @Override + public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precision) { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + + final double tiles = 1 << precision; + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + int idx = 0; + for (int i = minXTile; i <= maxXTile; i++) { + for (int j = minYTile; j <= maxYTile; j++) { + Rectangle rectangle = GeoTileUtils.toBoundingBox(i, j, precision); + if (geoValue.intersects(rectangle)) { + values[idx++] = GeoTileUtils.longEncodeTiles(precision, i, j); + } + } + } + + return idx; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java index d58beeb781c25..5830d2338227f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregationBuilder.java @@ -59,8 +59,8 @@ public GeoGridAggregationBuilder precision(int precision) { } @Override - protected ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + protected ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { return new GeoHashGridAggregatorFactory(name, config, precision, requiredSize, shardSize, queryShardContext, parent, diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java index a049a07f13dbe..178d4053e3800 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java @@ -19,7 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -28,7 +27,6 @@ import org.elasticsearch.search.aggregations.NonCollectingAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.GeoPoint; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.internal.SearchContext; @@ -38,13 +36,13 @@ import java.util.List; import java.util.Map; -public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory { +public class GeoHashGridAggregatorFactory extends ValuesSourceAggregatorFactory { private final int precision; private final int requiredSize; private final int shardSize; - GeoHashGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, + GeoHashGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); @@ -69,7 +67,7 @@ public InternalAggregation buildEmptyAggregation() { } @Override - protected Aggregator doCreateInternal(final GeoPoint valuesSource, + protected Aggregator doCreateInternal(final ValuesSource.Geo valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, @@ -78,8 +76,8 @@ protected Aggregator doCreateInternal(final GeoPoint valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, Geohash::longEncode); + CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, GeoGridTiler.GeoHashGridTiler.INSTANCE); return new GeoHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, searchContext, parent, - pipelineAggregators, metaData); + pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java index b3d9888781362..0ac6cebff2762 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregationBuilder.java @@ -58,8 +58,8 @@ public GeoGridAggregationBuilder precision(int precision) { } @Override - protected ValuesSourceAggregatorFactory createFactory( - String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, + protected ValuesSourceAggregatorFactory createFactory( + String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData ) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index 8380a4172c9c5..5645c56c57339 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -27,7 +27,6 @@ import org.elasticsearch.search.aggregations.NonCollectingAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSource.GeoPoint; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; import org.elasticsearch.search.internal.SearchContext; @@ -37,13 +36,13 @@ import java.util.List; import java.util.Map; -public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory { +public class GeoTileGridAggregatorFactory extends ValuesSourceAggregatorFactory { private final int precision; private final int requiredSize; private final int shardSize; - GeoTileGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, + GeoTileGridAggregatorFactory(String name, ValuesSourceConfig config, int precision, int requiredSize, int shardSize, QueryShardContext queryShardContext, AggregatorFactory parent, AggregatorFactories.Builder subFactoriesBuilder, Map metaData) throws IOException { super(name, config, queryShardContext, parent, subFactoriesBuilder, metaData); @@ -68,7 +67,7 @@ public InternalAggregation buildEmptyAggregation() { } @Override - protected Aggregator doCreateInternal(final GeoPoint valuesSource, + protected Aggregator doCreateInternal(final ValuesSource.Geo valuesSource, SearchContext searchContext, Aggregator parent, boolean collectsFromSingleBucket, @@ -77,8 +76,8 @@ protected Aggregator doCreateInternal(final GeoPoint valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, GeoTileUtils::longEncode); + CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, GeoGridTiler.GeoTileGridTiler.INSTANCE); return new GeoTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, searchContext, parent, - pipelineAggregators, metaData); + pipelineAggregators, metaData); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index c417be016288d..0f525e2ccdffc 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.geometry.Rectangle; import java.io.IOException; import java.util.Locale; @@ -88,6 +89,49 @@ public static int checkPrecisionRange(int precision) { return precision; } + /** + * Calculates the x-coordinate in the tile grid for the specified longitude given + * the number of tile columns for a pre-determined zoom-level. + * + * @param longitude the longitude to use when determining the tile x-coordinate + * @param tiles the number of tiles per row for a pre-determined zoom-level + */ + static int getXTile(double longitude, long tiles) { + int xTile = (int) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); + + // Edge values may generate invalid values, and need to be clipped. + // For example, polar regions (above/below lat 85.05112878) get normalized. + if (xTile < 0) { + return 0; + } + if (xTile >= tiles) { + return (int) tiles - 1; + } + + return xTile; + } + + /** + * Calculates the y-coordinate in the tile grid for the specified longitude given + * the number of tile rows for pre-determined zoom-level. + * + * @param latitude the latitude to use when determining the tile y-coordinate + * @param tiles the number of tiles per column for a pre-determined zoom-level + */ + static int getYTile(double latitude, long tiles) { + double latSin = Math.sin(Math.toRadians(normalizeLat(latitude))); + int yTile = (int) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); + + if (yTile < 0) { + yTile = 0; + } + if (yTile >= tiles) { + return (int) tiles - 1; + } + + return yTile; + } + /** * Encode lon/lat to the geotile based long format. * The resulting hash contains interleaved tile X and Y coordinates. @@ -119,7 +163,7 @@ public static long longEncode(double longitude, double latitude, int precision) yTile = tiles - 1; } - return longEncode((long) precision, xTile, yTile); + return longEncodeTiles(precision, xTile, yTile); } /** @@ -130,7 +174,14 @@ public static long longEncode(double longitude, double latitude, int precision) */ public static long longEncode(String hashAsString) { int[] parsed = parseHash(hashAsString); - return longEncode((long)parsed[0], (long)parsed[1], (long)parsed[2]); + return longEncode((long) parsed[0], (long) parsed[1], (long) parsed[2]); + } + + static long longEncodeTiles(int precision, long xTile, long yTile) { + // Zoom value is placed in front of all the bits used for the geotile + // e.g. when max zoom is 29, the largest index would use 58 bits (57th..0th), + // leaving 5 bits unused for zoom. See MAX_ZOOM comment above. + return ((long) precision << ZOOM_SHIFT) | (xTile << MAX_ZOOM) | yTile; } /** @@ -192,6 +243,19 @@ static GeoPoint keyToGeoPoint(String hashAsString) { return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]); } + static Rectangle toBoundingBox(int xTile, int yTile, int precision) { + final double tiles = validateZXY(precision, xTile, yTile); + final double minN = Math.PI - (2.0 * Math.PI * (yTile + 1)) / tiles; + final double maxN = Math.PI - (2.0 * Math.PI * (yTile)) / tiles; + final double minY = Math.toDegrees(Math.atan(Math.sinh(minN))); + final double minX = ((xTile) / tiles * 360.0) - 180; + + final double maxY = Math.toDegrees(Math.atan(Math.sinh(maxN))); + final double maxX = ((xTile + 1) / tiles * 360.0) - 180; + + return new Rectangle(minX, maxX, maxY, minY); + } + /** * Validates Zoom, X, and Y values, and returns the total number of allowed tiles along the x/y axis. */ diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java index 024a86d186ded..86ca9c0a79076 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/MissingValues.java @@ -417,6 +417,11 @@ public int docValueCount() { return count == 0 ? 1 : count; } + @Override + public ValuesSourceType valuesSourceType() { + return values.valuesSourceType(); + } + @Override public GeoValue nextValue() throws IOException { if (count > 0) { diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java index 720b775b0ec42..b8368c81b8d4b 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/ScriptDocValuesGeoPointsTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues.GeoPoints; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -54,6 +55,11 @@ public boolean advanceExact(int docId) { public int docValueCount() { return current.length; } + + @Override + public ValuesSourceType valuesSourceType() { + return ValuesSourceType.GEOPOINT; + } }; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 047903bc86100..13b9ee51d8ae3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.apache.lucene.document.Document; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.DirectoryReader; @@ -28,7 +29,11 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorTestCase; @@ -68,7 +73,13 @@ public void testNoDocs() throws IOException { // Intentionally not writing any docs }, geoGrid -> { assertEquals(0, geoGrid.getBuckets().size()); - }); + }, new GeoPointFieldMapper.GeoPointFieldType()); + + testCase(new MatchAllDocsQuery(), FIELD_NAME, randomPrecision(), iw -> { + // Intentionally not writing any docs + }, geoGrid -> { + assertEquals(0, geoGrid.getBuckets().size()); + }, new GeoShapeFieldMapper.GeoShapeFieldType()); } public void testFieldMissing() throws IOException { @@ -76,10 +87,16 @@ public void testFieldMissing() throws IOException { iw.addDocument(Collections.singleton(new LatLonDocValuesField(FIELD_NAME, 10D, 10D))); }, geoGrid -> { assertEquals(0, geoGrid.getBuckets().size()); - }); + }, new GeoPointFieldMapper.GeoPointFieldType()); + + testCase(new MatchAllDocsQuery(), "wrong_field", randomPrecision(), iw -> { + iw.addDocument(Collections.singleton(new BinaryGeoShapeDocValuesField(FIELD_NAME, new Point(10D, 10D)))); + }, geoGrid -> { + assertEquals(0, geoGrid.getBuckets().size()); + }, new GeoShapeFieldMapper.GeoShapeFieldType()); } - public void testWithSeveralDocs() throws IOException { + public void testGeoPointWithSeveralDocs() throws IOException { int precision = randomPrecision(); int numPoints = randomIntBetween(8, 128); Map expectedCountPerGeoHash = new HashMap<>(); @@ -118,11 +135,58 @@ public void testWithSeveralDocs() throws IOException { assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); } assertTrue(AggregationInspectionHelper.hasValue(geoHashGrid)); - }); + }, new GeoPointFieldMapper.GeoPointFieldType()); + } + + public void testGeoShapeWithSeveralDocs() throws IOException { + int precision = randomIntBetween(1, 4); + int numShapes = randomIntBetween(8, 128); + Map expectedCountPerGeoHash = new HashMap<>(); + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, iw -> { + List shapes = new ArrayList<>(); + Document document = new Document(); + Set distinctHashesPerDoc = new HashSet<>(); + for (int shapeId = 0; shapeId < numShapes; shapeId++) { + // undefined close to pole + double lat = (170.10225756d * randomDouble()) - 85.05112878d; + double lng = (360d * randomDouble()) - 180d; + + // Precision-adjust longitude/latitude to avoid wrong bucket placement + // Internally, lat/lng get converted to 32 bit integers, loosing some precision. + // This does not affect geohashing because geohash uses the same algorithm, + // but it does affect other bucketing algos, thus we need to do the same steps here. + lng = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lng)); + lat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); + + shapes.add(new Point(lng, lat)); + String hash = hashAsString(lng, lat, precision); + if (distinctHashesPerDoc.contains(hash) == false) { + expectedCountPerGeoHash.put(hash, expectedCountPerGeoHash.getOrDefault(hash, 0) + 1); + } + distinctHashesPerDoc.add(hash); + if (usually()) { + document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, new MultiPoint(new ArrayList<>(shapes)))); + iw.addDocument(document); + shapes.clear(); + distinctHashesPerDoc.clear(); + document.clear(); + } + } + if (shapes.size() != 0) { + document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, new MultiPoint(new ArrayList<>(shapes)))); + iw.addDocument(document); + } + }, geoHashGrid -> { + assertEquals(expectedCountPerGeoHash.size(), geoHashGrid.getBuckets().size()); + for (GeoGrid.Bucket bucket : geoHashGrid.getBuckets()) { + assertEquals((long) expectedCountPerGeoHash.get(bucket.getKeyAsString()), bucket.getDocCount()); + } + assertTrue(AggregationInspectionHelper.hasValue(geoHashGrid)); + }, new GeoShapeFieldMapper.GeoShapeFieldType()); } private void testCase(Query query, String field, int precision, CheckedConsumer buildIndex, - Consumer> verify) throws IOException { + Consumer> verify, MappedFieldType fieldType) throws IOException { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); buildIndex.accept(indexWriter); @@ -133,7 +197,6 @@ private void testCase(Query query, String field, int precision, CheckedConsumer< GeoGridAggregationBuilder aggregationBuilder = createBuilder("_name").field(field); aggregationBuilder.precision(precision); - MappedFieldType fieldType = new GeoPointFieldMapper.GeoPointFieldType(); fieldType.setHasDocValues(true); fieldType.setName(FIELD_NAME); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java new file mode 100644 index 0000000000000..82d1d69a49eb2 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -0,0 +1,110 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.GeometryTreeReader; +import org.elasticsearch.common.geo.GeometryTreeWriter; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +// TODO(talevy): more tests +public class GeoGridTilerTests extends ESTestCase { + private static final GeoGridTiler.GeoTileGridTiler GEOTILE = GeoGridTiler.GeoTileGridTiler.INSTANCE; + private static final GeoGridTiler.GeoHashGridTiler GEOHASH = GeoGridTiler.GeoHashGridTiler.INSTANCE; + + public void testGeoTile() throws Exception { + double x = randomDouble(); + double y = randomDouble(); + int precision = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + assertThat(GEOTILE.encode(x, y, precision), equalTo(GeoTileUtils.longEncode(x, y, precision))); + + // create rectangle within tile and check bound counts + Rectangle tile = GeoTileUtils.toBoundingBox(1309, 3166, 13); + GeometryTreeWriter writer = new GeometryTreeWriter( + new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, + tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001), GeoShapeCoordinateEncoder.INSTANCE); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + + long[] values = new long[16]; + + // test shape within tile bounds + { + int count = GEOTILE.setValues(values, value, 13); + assertThat(GEOTILE.getBoundingTileCount(value, 13), equalTo(1L)); + assertThat(count, equalTo(1)); + } + { + int count = GEOTILE.setValues(values, value, 14); + assertThat(GEOTILE.getBoundingTileCount(value, 14), equalTo(4L)); + assertThat(count, equalTo(4)); + } + { + int count = GEOTILE.setValues(values, value, 15); + assertThat(GEOTILE.getBoundingTileCount(value, 15), equalTo(16L)); + assertThat(count, equalTo(16)); + } + } + + public void testGeoHash() throws Exception { + double x = randomDouble(); + double y = randomDouble(); + int precision = randomIntBetween(0, Geohash.PRECISION); + assertThat(GEOHASH.encode(x, y, precision), equalTo(Geohash.longEncode(x, y, precision))); + + Rectangle tile = Geohash.toBoundingBox(Geohash.stringEncode(x, y, 5)); + + GeometryTreeWriter writer = new GeometryTreeWriter( + new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, + tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001), GeoShapeCoordinateEncoder.INSTANCE); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + + long[] values = new long[1024]; + + // test shape within tile bounds + { + int count = GEOHASH.setValues(values, value, 5); + assertThat(GEOHASH.getBoundingTileCount(value, 5), equalTo(1L)); + assertThat(count, equalTo(1)); + } + { + int count = GEOHASH.setValues(values, value, 6); + assertThat(GEOHASH.getBoundingTileCount(value, 6), equalTo(32L)); + assertThat(count, equalTo(32)); + } + { + int count = GEOHASH.setValues(values, value, 7); + assertThat(GEOHASH.getBoundingTileCount(value, 7), equalTo(1024L)); + assertThat(count, equalTo(1024)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index 6544344543e34..85b2306403230 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -46,5 +46,4 @@ public void testPrecision() { builder.precision(precision); assertEquals(precision, builder.precision()); } - } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index f0140c079a9a5..8d8946b716023 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -374,6 +374,11 @@ public boolean advanceExact(int docId) { public int docValueCount() { return values[doc].length; } + + @Override + public ValuesSourceType valuesSourceType() { + return ValuesSourceType.GEOPOINT; + } }; final MultiGeoValues.GeoPointValue missing = new MultiGeoValues.GeoPointValue( new GeoPoint(randomDouble() * 90, randomDouble() * 180)); From 71a5c84479eb8bdc1bdea851bdf0f1317e59150e Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 24 Oct 2019 15:33:08 -0700 Subject: [PATCH 23/62] fix geoshapefieldmappertests to expect docvalues --- .../elasticsearch/index/mapper/GeoShapeFieldMapperTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index e14d04eb7d664..280668c1c6dbc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -60,7 +60,7 @@ public void testDefaultConfiguration() throws IOException { GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value())); - assertThat(geoShapeFieldMapper.fieldType.hasDocValues(), equalTo(false)); + assertTrue(geoShapeFieldMapper.fieldType.hasDocValues()); } /** From 63cab7d458978aec183d7277a3920e9c56e1ab37 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 28 Oct 2019 07:57:03 -0700 Subject: [PATCH 24/62] fix compile issue in ValuesSourceConfigTests: type removal --- .../support/ValuesSourceConfigTests.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java index 42f9d16528d28..163126bfeeaba 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfigTests.java @@ -270,7 +270,7 @@ public void testUnmappedBoolean() throws Exception { public void testGeoPoint() throws IOException { IndexService indexService = createIndex("index", Settings.EMPTY, "type", "geo_point", "type=geo_point"); - client().prepareIndex("index", "type", "1") + client().prepareIndex("index") .setSource("geo_point", "-10.0,10.0") .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -294,7 +294,7 @@ public void testGeoPoint() throws IOException { public void testEmptyGeoPoint() throws IOException { IndexService indexService = createIndex("index", Settings.EMPTY, "type", "geo_point", "type=geo_point"); - client().prepareIndex("index", "type", "1") + client().prepareIndex("index") .setSource() .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -324,7 +324,7 @@ public void testEmptyGeoPoint() throws IOException { public void testUnmappedGeoPoint() throws IOException { IndexService indexService = createIndex("index", Settings.EMPTY, "type"); - client().prepareIndex("index", "type", "1") + client().prepareIndex("index") .setSource() .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -353,7 +353,7 @@ public void testUnmappedGeoPoint() throws IOException { public void testGeoShape() throws IOException { IndexService indexService = createIndex("index", Settings.EMPTY, "type", "geo_shape", "type=geo_shape"); - client().prepareIndex("index", "type", "1") + client().prepareIndex("index") .setSource("geo_shape", "POINT (-10 10)") .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -378,7 +378,7 @@ public void testGeoShape() throws IOException { public void testEmptyGeoShape() throws IOException { IndexService indexService = createIndex("index", Settings.EMPTY, "type", "geo_shape", "type=geo_shape"); - client().prepareIndex("index", "type", "1") + client().prepareIndex("index") .setSource() .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); @@ -414,7 +414,7 @@ public void testEmptyGeoShape() throws IOException { public void testUnmappedGeoShape() throws IOException { IndexService indexService = createIndex("index", Settings.EMPTY, "type"); - client().prepareIndex("index", "type", "1") + client().prepareIndex("index") .setSource() .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) .get(); From 0ad1b36a8d4269a2cad80d09b947d6f61ffea949 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 13 Nov 2019 14:47:41 -0800 Subject: [PATCH 25/62] fix rightOffset bug in EdgeTreeReader --- .../java/org/elasticsearch/common/geo/EdgeTreeReader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index beff33b761b97..e20fd8aa3ec67 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -144,7 +144,7 @@ private boolean containsBottomLeft(Edge root, Extent extent) throws IOException res ^= containsBottomLeft(readLeft(root), extent); } - if (root.rightOffset > 0 && extent.maxY() >= root.minY) { /* no right node if rightOffset == -1 */ + if (root.rightOffset >= 0 && extent.maxY() >= root.minY) { /* no right node if rightOffset == -1 */ res ^= containsBottomLeft(readRight(root), extent); } } @@ -175,7 +175,7 @@ && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX() res ^= containsBottomLeft(readLeft(root), extent); } - if (root.rightOffset > 0 && extent.maxY() >= root.minY) { /* no right node if rightOffset == -1 */ + if (root.rightOffset >= 0 && extent.maxY() >= root.minY) { /* no right node if rightOffset == -1 */ res ^= containsBottomLeft(readRight(root), extent); } } @@ -206,7 +206,7 @@ private boolean crosses(Edge root, Extent extent) throws IOException { } /* no right node if rightOffset == -1 */ - if (root.rightOffset > 0 && extent.maxY() >= root.minY && crosses(readRight(root), extent)) { + if (root.rightOffset >= 0 && extent.maxY() >= root.minY && crosses(readRight(root), extent)) { return true; } } From 27724ce96d4d81f4b2f3bf4d16bae27c8ab7d8f5 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 14 Nov 2019 18:03:11 -0500 Subject: [PATCH 26/62] Add randomized shape GeometryTree tests (#49112) The test is disabled at the moment since it fails regularly. Relates #37206 --- .../common/geo/GeometryTreeTests.java | 68 +++++++++++++++++-- .../elasticsearch/geo/GeometryTestUtils.java | 63 +++++++++++++++++ 2 files changed, 124 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index d8abe1aaadd69..da5c6aef63782 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Line; import org.elasticsearch.geometry.LinearRing; @@ -26,13 +27,17 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Function; +import static org.elasticsearch.geo.GeometryTestUtils.fold; +import static org.elasticsearch.geo.GeometryTestUtils.randomPoint; import static org.hamcrest.Matchers.equalTo; public class GeometryTreeTests extends ESTestCase { @@ -56,8 +61,8 @@ public void testRectangleShape() throws IOException { assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); // encoder loses precision when casting to integer, so centroid is calculated using integer division here - assertThat(reader.getCentroidX(), equalTo((double) ((minX + maxX)/2))); - assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY)/2))); + assertThat(reader.getCentroidX(), equalTo((double) ((minX + maxX) / 2))); + assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY) / 2))); // box-query touches bottom-left corner assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), minX, minY))); @@ -117,8 +122,8 @@ public void testPacManPolygon() throws Exception { // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon public void testPolygonWithHole() throws Exception { - Polygon polyWithHole = new Polygon(new LinearRing(new double[] { -50, 50, 50, -50, -50 }, new double[] { -50, -50, 50, 50, -50 }), - Collections.singletonList(new LinearRing(new double[] { -10, 10, 10, -10, -10 }, new double[] { -10, -10, 10, 10, -10 }))); + Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), + Collections.singletonList(new LinearRing(new double[]{-10, 10, 10, -10, -10}, new double[]{-10, -10, 10, 10, -10}))); GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); @@ -135,13 +140,13 @@ public void testPolygonWithHole() throws Exception { } public void testCombPolygon() throws Exception { - double[] px = {0, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 0, 0}; - double[] py = {0, 0, 20, 20, 0, 0, 20, 20, 0, 0, 30, 30, 0}; + double[] px = {0, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 0, 0}; + double[] py = {0, 0, 20, 20, 0, 0, 20, 20, 0, 0, 30, 30, 0}; double[] hx = {21, 21, 29, 29, 21}; double[] hy = {1, 20, 20, 1, 1}; - Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); + Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); // test cell crossing poly GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole, TestCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); @@ -217,4 +222,53 @@ public void testPacManPoints() throws Exception { GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(xMin, yMin, xMax, yMax))); } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") + public void testRandomGeometryIntersection() throws IOException { + int testPointCount = randomIntBetween(100, 200); + Point[] testPoints = new Point[testPointCount]; + double extentSize = randomDoubleBetween(1, 10, true); + boolean[] intersects = new boolean[testPointCount]; + for (int i = 0; i < testPoints.length; i++) { + testPoints[i] = randomPoint(false); + } + + Geometry geometry = randomGeometryTreeGeometry(); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + + for (int i = 0; i < testPointCount; i++) { + int cur = i; + intersects[cur] = fold(geometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); + } + + for (int i = 0; i < testPointCount; i++) { + assertEquals(intersects[i], intersects(geometry, testPoints[i], extentSize)); + } + } + + private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { + // TODO: Make this independent from GeometryTree + GeometryTreeWriter writer = new GeometryTreeWriter(g, GeoShapeCoordinateEncoder.INSTANCE); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + int xMin = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.max(p.getX() - extentSize, -180.0)); + int xMax = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.min(p.getX() + extentSize, 180.0)); + int yMin = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.max(p.getY() - extentSize, -90)); + int yMax = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.min(p.getY() + extentSize, 90)); + GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), GeoShapeCoordinateEncoder.INSTANCE); + return reader.intersects(Extent.fromPoints(xMin, yMin, xMax, yMax)); + } + + private static Geometry randomGeometryTreeGeometry() { + @SuppressWarnings("unchecked") Function geometry = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPoint + ); + return geometry.apply(false); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java b/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java index e4522a37e486a..5cf18929ec672 100644 --- a/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/geo/GeometryTestUtils.java @@ -20,6 +20,7 @@ package org.elasticsearch.geo; import org.apache.lucene.geo.GeoTestUtil; +import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; @@ -258,4 +259,66 @@ public MultiPoint visit(Rectangle rectangle) throws RuntimeException { } }); } + + /** + * Preforms left fold operation on all primitive geometries (points, lines polygons, circles and rectangles). + * All collection geometries are iterated depth first. + */ + public static R fold(Geometry geometry, R state, CheckedBiFunction operation) throws E { + return geometry.visit(new GeometryVisitor() { + @Override + public R visit(Circle circle) throws E { + return operation.apply(geometry, state); + } + + @Override + public R visit(GeometryCollection collection) throws E { + R ret = state; + for (Geometry g : collection) { + ret = fold(g, ret, operation); + } + return ret; + } + + @Override + public R visit(Line line) throws E { + return operation.apply(line, state); + } + + @Override + public R visit(LinearRing ring) throws E { + return operation.apply(ring, state); + } + + @Override + public R visit(MultiLine multiLine) throws E { + return visit((GeometryCollection) multiLine); + } + + @Override + public R visit(MultiPoint multiPoint) throws E { + return visit((GeometryCollection) multiPoint); } + + @Override + public R visit(MultiPolygon multiPolygon) throws E { + return visit((GeometryCollection) multiPolygon); + } + + @Override + public R visit(Point point) throws E { + return operation.apply(point, state); + } + + @Override + public R visit(Polygon polygon) throws E { + return operation.apply(polygon, state); + } + + @Override + public R visit(Rectangle rectangle) throws E { + return operation.apply(rectangle, state); + } + }); + } + } From 2131daa093d131735e378f6e68643ec388bec61f Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 15 Nov 2019 08:04:53 -0800 Subject: [PATCH 27/62] fix bug in EdgeTreeReader's intersect logic (#49127) EdgeTreeReader's intersect was claiming queries where the Extent fully encloses a Line of the MultiLine did not intersect with the overall shape. This commit fixes this and adds test for that scenario. relates #37206. --- .../common/geo/EdgeTreeReader.java | 7 ++ .../common/geo/GeometryTreeTests.java | 107 ++++++++++-------- 2 files changed, 67 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index e20fd8aa3ec67..2d5d3083d99e7 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -200,6 +200,13 @@ private boolean crosses(Edge root, Extent extent) throws IOException { extent.minX(), extent.maxY(), extent.minX(), extent.minY())) { return true; } + + // does this edge fully reside within the rectangle's area + if (extent.minX() <= Math.min(root.x1, root.x2) && extent.minY() <= Math.min(root.y1, root.y2) + && extent.maxX() >= Math.max(root.x1, root.x2) && extent.maxY() >= Math.max(root.y1, root.y2)) { + return true; + } + /* has left node */ if (root.rightOffset > 0 && crosses(readLeft(root), extent)) { return true; diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index da5c6aef63782..b897ebd118519 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Line; import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; @@ -52,12 +53,7 @@ public void testRectangleShape() throws IOException { double[] y = new double[]{minY, minY, maxY, maxY, minY}; Geometry rectangle = randomBoolean() ? new Polygon(new LinearRing(x, y), Collections.emptyList()) : new Rectangle(minX, maxX, maxY, minY); - GeometryTreeWriter writer = new GeometryTreeWriter(rectangle, TestCoordinateEncoder.INSTANCE); - - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); + GeometryTreeReader reader = geometryTreeReader(rectangle, TestCoordinateEncoder.INSTANCE); assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); // encoder loses precision when casting to integer, so centroid is calculated using integer division here @@ -108,12 +104,8 @@ public void testPacManPolygon() throws Exception { double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(new Polygon(new LinearRing(py, px), Collections.emptyList()), + GeometryTreeReader reader = geometryTreeReader(new Polygon(new LinearRing(py, px), Collections.emptyList()), TestCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); @@ -125,11 +117,7 @@ public void testPolygonWithHole() throws Exception { Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), Collections.singletonList(new LinearRing(new double[]{-10, 10, 10, -10, -10}, new double[]{-10, -10, 10, 10, -10}))); - GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole, TestCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), null); + GeometryTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); assertFalse(reader.intersects(Extent.fromPoints(6, -6, 6, -6))); // in the hole assertTrue(reader.intersects(Extent.fromPoints(25, -25, 25, -25))); // on the mainland @@ -147,12 +135,8 @@ public void testCombPolygon() throws Exception { double[] hy = {1, 20, 20, 1, 1}; Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); + GeometryTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(polyWithHole, TestCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(5, 10, 5, 10))); assertFalse(reader.intersects(Extent.fromPoints(15, 10, 15, 10))); assertFalse(reader.intersects(Extent.fromPoints(25, 10, 25, 10))); @@ -164,11 +148,7 @@ public void testPacManClosedLineString() throws Exception { double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(new Line(px, py), TestCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); + GeometryTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); @@ -181,11 +161,7 @@ public void testPacManLineString() throws Exception { double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(new Line(px, py), TestCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); + GeometryTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); @@ -215,14 +191,44 @@ public void testPacManPoints() throws Exception { int yMax = 9; // test cell crossing poly - GeometryTreeWriter writer = new GeometryTreeWriter(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), TestCoordinateEncoder.INSTANCE); + GeometryTreeReader reader = geometryTreeReader(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); assertTrue(reader.intersects(Extent.fromPoints(xMin, yMin, xMax, yMax))); } + public void testRandomMultiLineIntersections() throws IOException { + double extentSize = randomDoubleBetween(0.01, 10, true); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + MultiLine geometry = GeometryTestUtils.randomMultiLine(false); + geometry = (MultiLine) indexer.prepareForIndexing(geometry); + + GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + Extent readerExtent = reader.getExtent(); + + for (Line line : geometry) { + // extent that intersects edges + assertTrue(reader.intersects(bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize))); + + // extent that fully encloses a line in the MultiLine + Extent lineExtent = geometryTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); + assertTrue(reader.intersects(lineExtent)); + + if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE + && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { + assertTrue(reader.intersects(Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, + lineExtent.maxX() + 1, lineExtent.maxY() + 1))); + } + } + + // extent that fully encloses the MultiLine + assertTrue(reader.intersects(reader.getExtent())); + if (readerExtent.minX() != Integer.MIN_VALUE && readerExtent.maxX() != Integer.MAX_VALUE + && readerExtent.minY() != Integer.MIN_VALUE && readerExtent.maxY() != Integer.MAX_VALUE) { + assertTrue(reader.intersects(Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, + readerExtent.maxX() + 1, readerExtent.maxY() + 1))); + } + + } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") public void testRandomGeometryIntersection() throws IOException { int testPointCount = randomIntBetween(100, 200); @@ -247,18 +253,17 @@ public void testRandomGeometryIntersection() throws IOException { } } + private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) { + int xMin = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.max(x - extentSize, -180.0)); + int xMax = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.min(x + extentSize, 180.0)); + int yMin = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.max(y - extentSize, -90)); + int yMax = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.min(y + extentSize, 90)); + return Extent.fromPoints(xMin, yMin, xMax, yMax); + } + private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { - // TODO: Make this independent from GeometryTree - GeometryTreeWriter writer = new GeometryTreeWriter(g, GeoShapeCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - int xMin = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.max(p.getX() - extentSize, -180.0)); - int xMax = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.min(p.getX() + extentSize, 180.0)); - int yMin = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.max(p.getY() - extentSize, -90)); - int yMax = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.min(p.getY() + extentSize, 90)); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), GeoShapeCoordinateEncoder.INSTANCE); - return reader.intersects(Extent.fromPoints(xMin, yMin, xMax, yMax)); + return geometryTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) + .intersects(bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize)); } private static Geometry randomGeometryTreeGeometry() { @@ -271,4 +276,12 @@ private static Geometry randomGeometryTreeGeometry() { ); return geometry.apply(false); } + + private GeometryTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { + GeometryTreeWriter writer = new GeometryTreeWriter(geometry, encoder); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + return new GeometryTreeReader(output.bytes().toBytesRef(), encoder); + } } From 84e69ddb228aab5fe02ce2bfc73889727ccbef1f Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 15 Nov 2019 12:59:52 -0800 Subject: [PATCH 28/62] Add disjointness check for parallel lines in EdgeTreeReader (#49188) * Add disjointness check for parallel lines GeoUtils#lineCrossesLineWithBoundary returns true for parallel lines that are disjoint. This commit adds another disjointness check for each edge to verify that they should not be accounted for even if lineCrossesLine claims they cross. relates #37206. --- .../elasticsearch/common/geo/EdgeTreeReader.java | 13 +++++++++++-- .../elasticsearch/common/geo/GeometryTreeTests.java | 12 ++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index 2d5d3083d99e7..55147a2759264 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -189,15 +189,24 @@ private boolean crosses(Edge root, Extent extent) throws IOException { // we just have to cross one edge to answer the question, so we descend the tree and return when we do. if (root.maxY >= extent.minY()) { + double a1x = root.x1; + double a1y = root.y1; + double b1x = root.x2; + double b1y = root.y2; + boolean outside = (a1y < extent.minY() && b1y < extent.minY()) || + (a1y > extent.maxY() && b1y > extent.maxY()) || + (a1x < extent.minX() && b1x < extent.minX()) || + (a1x > extent.maxX() && b1x > extent.maxX()); + // does rectangle's edges intersect or reside inside polygon's edge - if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + if (outside == false && (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX(), extent.minY(), extent.maxX(), extent.minY()) || lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX(), extent.minY(), extent.maxX(), extent.maxY()) || lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX(), extent.maxY(), extent.minX(), extent.maxY()) || lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, - extent.minX(), extent.maxY(), extent.minX(), extent.minY())) { + extent.minX(), extent.maxY(), extent.minX(), extent.minY()))) { return true; } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index b897ebd118519..7b47f5bde65d4 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.test.ESTestCase; @@ -229,7 +230,6 @@ public void testRandomMultiLineIntersections() throws IOException { } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") public void testRandomGeometryIntersection() throws IOException { int testPointCount = randomIntBetween(100, 200); Point[] testPoints = new Point[testPointCount]; @@ -241,15 +241,19 @@ public void testRandomGeometryIntersection() throws IOException { Geometry geometry = randomGeometryTreeGeometry(); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - geometry = indexer.prepareForIndexing(geometry); + Geometry preparedGeometry = indexer.prepareForIndexing(geometry); + + // TODO: support multi-polygons + assumeFalse("polygon crosses dateline", + ShapeType.POLYGON == geometry.type() && ShapeType.MULTIPOLYGON == preparedGeometry.type()); for (int i = 0; i < testPointCount; i++) { int cur = i; - intersects[cur] = fold(geometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); + intersects[cur] = fold(preparedGeometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); } for (int i = 0; i < testPointCount; i++) { - assertEquals(intersects[i], intersects(geometry, testPoints[i], extentSize)); + assertEquals(intersects[i], intersects(preparedGeometry, testPoints[i], extentSize)); } } From 65c23eee88b29eaee3f63f8ac8d34274cd5bc451 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 21 Nov 2019 10:57:24 -0800 Subject: [PATCH 29/62] re-introduce custom GEO logic for missing values --- .../support/CoreValuesSourceType.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index 149686fc448da..5064de8b76575 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.support; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -230,28 +231,32 @@ public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, GEO { @Override public ValuesSource getEmpty() { - return ValuesSource.GeoShape.EMPTY; + return ValuesSource.Geo.EMPTY; } @Override public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { - throw new AggregationExecutionException("value source of type [" + this.value() + "] is not supported by scripts"); + throw new UnsupportedOperationException("CoreValuesSourceType.GEO is still a special case"); } @Override public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - if (!(fieldContext.indexFieldData() instanceof IndexGeoShapeFieldData)) { - // TODO: Is this the correct exception type here? - throw new IllegalArgumentException("Expected geo_shape type on field [" + fieldContext.field() + - "], but got [" + fieldContext.fieldType().typeName() + "]"); - } - - return new ValuesSource.GeoShape.Fielddata((IndexGeoShapeFieldData) fieldContext.indexFieldData()); + throw new UnsupportedOperationException("CoreValuesSourceType.GEO is still a special case"); } @Override public ValuesSource replaceMissing(ValuesSource valuesSource, Object rawMissing, DocValueFormat docValueFormat, LongSupplier now) { - throw new UnsupportedOperationException("CoreValuesSourceType.GEO is still a special case"); + // when missing value is present on aggregations that support both shapes and points, geo_point will be + // assumed first to preserve backwards compatibility with existing behavior. If a value is not a valid geo_point + // then it is parsed as a geo_shape + try { + final MultiGeoValues.GeoPointValue missing = new + MultiGeoValues.GeoPointValue(new GeoPoint(rawMissing.toString())); + return MissingValues.replaceMissing(ValuesSource.GeoPoint.EMPTY, missing); + } catch (ElasticsearchParseException e) { + return MissingValues.replaceMissing(ValuesSource.GeoShape.EMPTY, + MultiGeoValues.GeoShapeValue.missing(rawMissing.toString())); + } } } ; From 72d0295d34b3ab0958168787bf4b4d8f7852662d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 21 Nov 2019 11:38:52 -0800 Subject: [PATCH 30/62] fix resolution of GEO field type in CoreValuesSourceType --- .../aggregations/support/CoreValuesSourceType.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java index 5064de8b76575..3659296ed5a50 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/CoreValuesSourceType.java @@ -236,12 +236,22 @@ public ValuesSource getEmpty() { @Override public ValuesSource getScript(AggregationScript.LeafFactory script, ValueType scriptValueType) { + // TODO (support scripts) throw new UnsupportedOperationException("CoreValuesSourceType.GEO is still a special case"); } @Override public ValuesSource getField(FieldContext fieldContext, AggregationScript.LeafFactory script) { - throw new UnsupportedOperationException("CoreValuesSourceType.GEO is still a special case"); + boolean isGeoPoint = fieldContext.indexFieldData() instanceof IndexGeoPointFieldData; + boolean isGeoShape = fieldContext.indexFieldData() instanceof IndexGeoShapeFieldData; + if (isGeoPoint == false && isGeoShape == false) { + throw new IllegalArgumentException("Expected geo_point or geo_shape type on field [" + fieldContext.field() + + "], but got [" + fieldContext.fieldType().typeName() + "]"); + } + if (isGeoPoint) { + return new ValuesSource.GeoPoint.Fielddata((IndexGeoPointFieldData) fieldContext.indexFieldData()); + } + return new ValuesSource.GeoShape.Fielddata((IndexGeoShapeFieldData) fieldContext.indexFieldData()); } @Override From b560af1188265fa4ce881e4270e5c9cafc71e37e Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 21 Nov 2019 12:47:25 -0800 Subject: [PATCH 31/62] remove unused geoshapeField --- .../aggregations/support/ValuesSourceConfig.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java index b84b074cf4f92..33e15d1866a8a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/ValuesSourceConfig.java @@ -288,14 +288,4 @@ public VS toValuesSource(QueryShardContext context, Function Date: Fri, 22 Nov 2019 07:27:42 -0800 Subject: [PATCH 32/62] Rasterize shape during geotilegrid computation (#49065) This PR mainly modifies the existing GeoTileGridTiler to rasterize the GeometryTree instead of iterating through all the tiles found in the bounding box of the shape. This PR also fixes a bug where containsFully was not being calculated correctly and simplifies all the relating logic to one `relate` method relates #37206. --- .../common/geo/EdgeTreeReader.java | 58 ++---- .../elasticsearch/common/geo/GeoRelation.java | 29 +++ .../common/geo/GeometryTreeReader.java | 41 +++-- .../common/geo/Point2DReader.java | 9 +- .../common/geo/PolygonTreeReader.java | 13 +- .../common/geo/ShapeTreeReader.java | 2 +- .../index/fielddata/MultiGeoValues.java | 16 +- .../bucket/geogrid/GeoGridTiler.java | 66 ++++++- .../bucket/geogrid/GeoTileUtils.java | 5 + .../common/geo/EdgeTreeTests.java | 68 ++++---- .../common/geo/GeoTestUtils.java | 55 ++++++ .../common/geo/GeometryTreeTests.java | 165 ++++++++++++------ .../common/geo/Point2DTests.java | 23 ++- .../bucket/geogrid/GeoGridTilerTests.java | 68 +++++++- .../bucket/geogrid/GeoTileUtilsTests.java | 22 +++ 15 files changed, 471 insertions(+), 169 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/GeoRelation.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index 55147a2759264..46a2bb49f4332 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -33,6 +33,9 @@ public class EdgeTreeReader implements ShapeTreeReader { private final ByteBufferStreamInput input; private final int startPosition; private final boolean hasArea; + private static final Optional OPTIONAL_FALSE = Optional.of(false); + private static final Optional OPTIONAL_TRUE = Optional.of(true); + private static final Optional OPTIONAL_EMPTY = Optional.empty(); public EdgeTreeReader(ByteBufferStreamInput input, boolean hasArea) throws IOException { this.startPosition = input.position(); @@ -48,25 +51,27 @@ public Extent getExtent() throws IOException { /** * Returns true if the rectangle query and the edge tree's shape overlap */ - public boolean intersects(Extent extent) throws IOException { - if (hasArea) { - return containsBottomLeft(extent) || crosses(extent); - } else { - return crosses(extent); + @Override + public GeoRelation relate(Extent extent) throws IOException { + if (crosses(extent)) { + return GeoRelation.QUERY_CROSSES; + } else if (hasArea && containsBottomLeft(extent)){ + return GeoRelation.QUERY_INSIDE; } + return GeoRelation.QUERY_DISJOINT; } static Optional checkExtent(Extent treeExtent, Extent extent) throws IOException { if (treeExtent.minY() > extent.maxY() || treeExtent.maxX() < extent.minX() || treeExtent.maxY() < extent.minY() || treeExtent.minX() > extent.maxX()) { - return Optional.of(false); // tree and bbox-query are disjoint + return OPTIONAL_FALSE; // tree and bbox-query are disjoint } if (extent.minX() <= treeExtent.minX() && extent.minY() <= treeExtent.minY() && extent.maxX() >= treeExtent.maxX() && extent.maxY() >= treeExtent.maxY()) { - return Optional.of(true); // bbox-query fully contains tree's extent. + return OPTIONAL_TRUE; // bbox-query fully contains tree's extent. } - return Optional.empty(); + return OPTIONAL_EMPTY; } boolean containsBottomLeft(Extent extent) throws IOException { @@ -80,12 +85,6 @@ boolean containsBottomLeft(Extent extent) throws IOException { return containsBottomLeft(readRoot(input.position()), extent); } - boolean containsFully(Extent extent) throws IOException { - resetInputPosition(); - input.position(input.position() + Extent.WRITEABLE_SIZE_IN_BYTES); // skip extent - return containsFully(readRoot(input.position()), extent); - } - public boolean crosses(Extent extent) throws IOException { resetInputPosition(); @@ -151,37 +150,6 @@ private boolean containsBottomLeft(Edge root, Extent extent) throws IOException return res; } - /** - * Returns true if every corner in the rectangle query is contained within the tree's edges. - */ - private boolean containsFully(Edge root, Extent extent) throws IOException { - boolean res = false; - if (root.maxY >= extent.minY()) { - // is bbox-query contained within linearRing - // cast infinite ray to the right from each corner of the extent - if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX(), extent.minY(), - Integer.MAX_VALUE, extent.minY()) - && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX(), extent.maxY(), - Integer.MAX_VALUE, extent.maxY()) - && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX(), extent.minY(), - Integer.MAX_VALUE, extent.minY()) - && lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.maxX(), extent.maxY(), - Integer.MAX_VALUE, extent.maxY()) - ) { - res = true; - } - - if (root.rightOffset > 0) { /* has left node */ - res ^= containsBottomLeft(readLeft(root), extent); - } - - if (root.rightOffset >= 0 && extent.maxY() >= root.minY) { /* no right node if rightOffset == -1 */ - res ^= containsBottomLeft(readRight(root), extent); - } - } - return res; - } - /** * Returns true if the box crosses any edge in this edge subtree * */ diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoRelation.java b/server/src/main/java/org/elasticsearch/common/geo/GeoRelation.java new file mode 100644 index 0000000000000..ac8f17d3ca81b --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoRelation.java @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +/** + * Enum for capturing relationships between a shape + * and a query + */ +public enum GeoRelation { + QUERY_CROSSES, + QUERY_INSIDE, + QUERY_DISJOINT +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 364ee63ce1ba4..5cb609be59f39 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -32,7 +32,7 @@ * This class supports checking bounding box * relations against the serialized geometry tree. */ -public class GeometryTreeReader { +public class GeometryTreeReader implements ShapeTreeReader { private final int extentOffset = 8; private final ByteBufferStreamInput input; @@ -43,6 +43,17 @@ public GeometryTreeReader(BytesRef bytesRef, CoordinateEncoder coordinateEncoder this.coordinateEncoder = coordinateEncoder; } + public double getCentroidX() throws IOException { + input.position(0); + return coordinateEncoder.decodeX(input.readInt()); + } + + public double getCentroidY() throws IOException { + input.position(4); + return coordinateEncoder.decodeY(input.readInt()); + } + + @Override public Extent getExtent() throws IOException { input.position(extentOffset); Extent extent = input.readOptionalWriteable(Extent::new); @@ -55,23 +66,15 @@ public Extent getExtent() throws IOException { return reader.getExtent(); } - public double getCentroidX() throws IOException { - input.position(0); - return coordinateEncoder.decodeX(input.readInt()); - } - - public double getCentroidY() throws IOException { - input.position(4); - return coordinateEncoder.decodeY(input.readInt()); - } - - public boolean intersects(Extent extent) throws IOException { + @Override + public GeoRelation relate(Extent extent) throws IOException { + GeoRelation relation = GeoRelation.QUERY_DISJOINT; input.position(extentOffset); boolean hasExtent = input.readBoolean(); if (hasExtent) { Optional extentCheck = EdgeTreeReader.checkExtent(new Extent(input), extent); if (extentCheck.isPresent()) { - return extentCheck.get(); + return extentCheck.get() ? GeoRelation.QUERY_INSIDE : GeoRelation.QUERY_DISJOINT; } } @@ -79,11 +82,17 @@ public boolean intersects(Extent extent) throws IOException { for (int i = 0; i < numTrees; i++) { ShapeType shapeType = input.readEnum(ShapeType.class); ShapeTreeReader reader = getReader(shapeType, input); - if (reader.intersects(extent)) { - return true; + GeoRelation shapeRelation = reader.relate(extent); + if (GeoRelation.QUERY_CROSSES == shapeRelation || + (GeoRelation.QUERY_DISJOINT == shapeRelation && GeoRelation.QUERY_INSIDE == relation) + ) { + return GeoRelation.QUERY_CROSSES; + } else { + relation = shapeRelation; } } - return false; + + return relation; } private static ShapeTreeReader getReader(ShapeType shapeType, ByteBufferStreamInput input) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java index 9ab106be76092..1fe9cb39faef8 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java @@ -50,7 +50,8 @@ public Extent getExtent() throws IOException { } } - public boolean intersects(Extent extent) throws IOException { + @Override + public GeoRelation relate(Extent extent) throws IOException { Deque stack = new ArrayDeque<>(); stack.push(0); stack.push(size - 1); @@ -66,7 +67,7 @@ public boolean intersects(Extent extent) throws IOException { int x = readX(i); int y = readY(i); if (x >= extent.minX() && x <= extent.maxX() && y >= extent.minY() && y <= extent.maxY()) { - return true; + return GeoRelation.QUERY_CROSSES; } } continue; @@ -76,7 +77,7 @@ public boolean intersects(Extent extent) throws IOException { int x = readX(middle); int y = readY(middle); if (x >= extent.minX() && x <= extent.maxX() && y >= extent.minY() && y <= extent.maxY()) { - return true; + return GeoRelation.QUERY_CROSSES; } if ((axis == 0 && extent.minX() <= x) || (axis == 1 && extent.minY() <= y)) { stack.push(left); @@ -90,7 +91,7 @@ public boolean intersects(Extent extent) throws IOException { } } - return false; + return GeoRelation.QUERY_DISJOINT; } private int readX(int pointIdx) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java index bfeaabc402e64..663e771cde67d 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java @@ -51,13 +51,16 @@ public Extent getExtent() throws IOException { * Returns true if the rectangle query and the edge tree's shape overlap */ @Override - public boolean intersects(Extent extent) throws IOException { + public GeoRelation relate(Extent extent) throws IOException { if (holes != null) { - boolean onlyInHole = holes.containsFully(extent); - if (onlyInHole) { - return false; + GeoRelation relation = holes.relate(extent); + if (GeoRelation.QUERY_CROSSES == relation) { + return GeoRelation.QUERY_CROSSES; + } + if (GeoRelation.QUERY_INSIDE == relation) { + return GeoRelation.QUERY_DISJOINT; } } - return outerShell.intersects(extent); + return outerShell.relate(extent); } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java index 94098d62bb5c5..0d5fb2a6d6b2d 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java @@ -28,5 +28,5 @@ public interface ShapeTreeReader { Extent getExtent() throws IOException; - boolean intersects(Extent extent) throws IOException; + GeoRelation relate(Extent extent) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index bb2290738d862..387555b34656c 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -18,9 +18,11 @@ */ package org.elasticsearch.index.fielddata; +import org.apache.lucene.spatial.util.GeoRelationUtils; import org.elasticsearch.common.geo.CoordinateEncoder; import org.elasticsearch.common.geo.Extent; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoRelation; import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.GeometryTreeReader; import org.elasticsearch.common.geo.GeometryTreeWriter; @@ -96,8 +98,12 @@ public BoundingBox boundingBox() { } @Override - public boolean intersects(Rectangle rectangle) { - throw new UnsupportedOperationException("intersect is unsupported for geo_point doc values"); + public GeoRelation relate(Rectangle rectangle) { + if (GeoRelationUtils.pointInRectPrecise(geoPoint.lat(), geoPoint.lon(), + rectangle.getMinLat(), rectangle.getMaxLat(), rectangle.getMinLon(), rectangle.getMaxLon())) { + return GeoRelation.QUERY_CROSSES; + } + return GeoRelation.QUERY_DISJOINT; } @Override @@ -141,14 +147,14 @@ public BoundingBox boundingBox() { * @return the latitude of the centroid of the shape */ @Override - public boolean intersects(Rectangle rectangle) { + public GeoRelation relate(Rectangle rectangle) { int minX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMinX()); int maxX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMaxX()); int minY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMinY()); int maxY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMaxY()); Extent extent = new Extent(maxY, minY, minX, maxX, minX, maxX); try { - return reader.intersects(extent); + return reader.relate(extent); } catch (IOException e) { throw new IllegalStateException("unable to check intersection", e); } @@ -194,7 +200,7 @@ public interface GeoValue { double lat(); double lon(); BoundingBox boundingBox(); - boolean intersects(Rectangle rectangle); + GeoRelation relate(Rectangle rectangle); } public static class BoundingBox { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java index aef09b458630c..3741a87384cb7 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.elasticsearch.common.geo.GeoRelation; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.fielddata.MultiGeoValues; @@ -88,7 +89,8 @@ public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precis for (double i = geoHashCell.getMinX(); i < bounds.maxX(); i+= Geohash.lonWidthInDegrees(precision)) { for (double j = geoHashCell.getMinY(); j < bounds.maxY(); j += Geohash.latHeightInDegrees(precision)) { Rectangle rectangle = Geohash.toBoundingBox(Geohash.stringEncode(i, j, precision)); - if (geoValue.intersects(rectangle)) { + GeoRelation relation = geoValue.relate(rectangle); + if (relation != GeoRelation.QUERY_DISJOINT) { values[idx++] = encode(i, j, precision); } } @@ -121,6 +123,17 @@ public long getBoundingTileCount(MultiGeoValues.GeoValue geoValue, int precision @Override public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precision) { + return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue); + } + + /** + * + * @param values the bucket values as longs + * @param geoValue the shape value + * @param precision the target precision to split the shape up into + * @return the number of buckets the geoValue is found in + */ + public int setValuesByBruteForceScan(long[] values, MultiGeoValues.GeoValue geoValue, int precision) { MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); final double tiles = 1 << precision; @@ -132,7 +145,7 @@ public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precis for (int i = minXTile; i <= maxXTile; i++) { for (int j = minYTile; j <= maxYTile; j++) { Rectangle rectangle = GeoTileUtils.toBoundingBox(i, j, precision); - if (geoValue.intersects(rectangle)) { + if (geoValue.relate(rectangle) != GeoRelation.QUERY_DISJOINT) { values[idx++] = GeoTileUtils.longEncodeTiles(precision, i, j); } } @@ -140,5 +153,54 @@ public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precis return idx; } + + private int setValuesByRasterization(int xTile, int yTile, int zTile, long[] values, int valuesIndex, int targetPrecision, + MultiGeoValues.GeoValue geoValue) { + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, zTile); + MultiGeoValues.BoundingBox shapeBounds = geoValue.boundingBox(); + if (shapeBounds.minX() == rectangle.getMaxX() || + shapeBounds.maxY() == rectangle.getMinY()) { + return valuesIndex; + } + GeoRelation relation = geoValue.relate(rectangle); + if (zTile == targetPrecision) { + if (GeoRelation.QUERY_DISJOINT != relation) { + values[valuesIndex++] = GeoTileUtils.longEncodeTiles(zTile, xTile, yTile); + } + return valuesIndex; + } + + if (GeoRelation.QUERY_INSIDE == relation) { + return setValuesForFullyContainedTile(xTile, yTile, zTile, values, valuesIndex, targetPrecision); + } + if (GeoRelation.QUERY_CROSSES == relation) { + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + valuesIndex = setValuesByRasterization(nextX, nextY, zTile + 1, values, valuesIndex, targetPrecision, geoValue); + } + } + } + + return valuesIndex; + } + + private int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, long[] values, int valuesIndex, int targetPrecision) { + if (zTile == targetPrecision) { + values[valuesIndex] = GeoTileUtils.longEncodeTiles(zTile, xTile, yTile); + return valuesIndex + 1; + } + + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile + 1, values, valuesIndex, targetPrecision); + } + } + + return valuesIndex; + } } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index f636e6bbc97b6..0dbdba6c23fff 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -98,6 +98,11 @@ public static int checkPrecisionRange(int precision) { * @param tiles the number of tiles per row for a pre-determined zoom-level */ static int getXTile(double longitude, long tiles) { + // normalizeLon treats this as 180, which is not friendly for tile mapping + if (longitude == -180) { + return 0; + } + int xTile = (int) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); // Edge values may generate invalid values, and need to be clipped. diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index 4bdeba2e55612..42bb45cc9ba7a 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -35,6 +35,7 @@ import java.nio.ByteBuffer; import java.util.List; +import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; import static org.hamcrest.Matchers.equalTo; public class EdgeTreeTests extends ESTestCase { @@ -57,45 +58,50 @@ public void testRectangleShape() throws IOException { assertThat(writer.getCentroidCalculator().getX(), equalTo((minX + maxX)/2.0)); assertThat(writer.getCentroidCalculator().getY(), equalTo((minY + maxY)/2.0)); + // box-query touches bottom-left corner - assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), minX, minY))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), + minY - randomIntBetween(1, 180), minX, minY)); // box-query touches bottom-right corner - assertTrue(reader.intersects(Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), minY))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), + maxX + randomIntBetween(1, 180), minY)); // box-query touches top-right corner - assertTrue(reader.intersects(Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), + maxY + randomIntBetween(1, 180))); // box-query touches top-left corner - assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, maxY + randomIntBetween(1, 180)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, + maxY + randomIntBetween(1, 180))); // box-query fully-enclosed inside rectangle - assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, - (3 * maxY + minY) / 4))); + assertRelation(GeoRelation.QUERY_INSIDE,reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); // box-query fully-contains poly - assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 180), minY - randomIntBetween(1, 180), - maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), + minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); // box-query half-in-half-out-right - assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), - (3 * maxY + minY) / 4))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); // box-query half-in-half-out-left - assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, - (3 * maxY + minY) / 4))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, + (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); // box-query half-in-half-out-top - assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 1000), - maxY + randomIntBetween(1, 1000)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + maxX + randomIntBetween(1, 1000), maxY + randomIntBetween(1, 1000))); // box-query half-in-half-out-bottom - assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); // box-query outside to the right - assertFalse(reader.intersects(Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, - maxX + randomIntBetween(1001, 2000), maxY))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, + maxX + randomIntBetween(1001, 2000), maxY)); // box-query outside to the left - assertFalse(reader.intersects(Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, - minX - randomIntBetween(1, 1000), maxY))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, + minX - randomIntBetween(1, 1000), maxY)); // box-query outside to the top - assertFalse(reader.intersects(Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, - maxY + randomIntBetween(1001, 2000)))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, + maxY + randomIntBetween(1001, 2000))); // box-query outside to the bottom - assertFalse(reader.intersects(Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, - minY - randomIntBetween(1, 1000)))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, + minY - randomIntBetween(1, 1000))); } } @@ -134,16 +140,16 @@ public void testSimplePolygon() throws IOException { assertThat(actualExtent.minY(), equalTo(minYBox)); assertThat(actualExtent.maxY(), equalTo(maxYBox)); // polygon fully contained within box - assertTrue(reader.intersects(Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox))); - // intersects + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox)); + // relate if (maxYBox - 1 >= minYBox) { - assertTrue(reader.intersects(Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1)); } if (maxXBox -1 >= minXBox) { - assertTrue(reader.intersects(Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox)); } // does not cross - assertFalse(reader.intersects(Extent.fromPoints(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10)); } } @@ -152,7 +158,7 @@ public void testPacMan() throws Exception { double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - // candidate intersects cell + // candidate relate cell int xMin = 2;//-5; int xMax = 11;//0.000001; int yMin = -1;//0; @@ -164,7 +170,7 @@ public void testPacMan() throws Exception { writer.writeTo(output); output.close(); EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); - assertTrue(reader.containsBottomLeft(Extent.fromPoints(xMin, yMin, xMax, yMax))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(xMin, yMin, xMax, yMax)); } public void testGetShapeType() { diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java new file mode 100644 index 0000000000000..f294d075ba34b --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -0,0 +1,55 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.geometry.Geometry; + +import java.io.IOException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +public class GeoTestUtils { + + public static void assertRelation(GeoRelation expectedRelation, ShapeTreeReader reader, Extent extent) throws IOException { + GeoRelation actualRelation = reader.relate(extent); + assertThat(actualRelation, equalTo(expectedRelation)); + } + + public static GeometryTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { + GeometryTreeWriter writer = new GeometryTreeWriter(geometry, encoder); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + return new GeometryTreeReader(output.bytes().toBytesRef(), encoder); + } + + public static String toGeoJsonString(Geometry geometry) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + GeoJson.toXContent(geometry, builder, ToXContent.EMPTY_PARAMS); + return XContentHelper.convertToJson(BytesReference.bytes(builder), true, false, XContentType.JSON); + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 7b47f5bde65d4..02c1f031ad41e 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.common.geo; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; @@ -25,19 +27,24 @@ import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.MultiLine; import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor; +import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.geo.RandomShapeGenerator; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.function.Function; +import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; import static org.elasticsearch.geo.GeometryTestUtils.fold; import static org.elasticsearch.geo.GeometryTestUtils.randomPoint; import static org.hamcrest.Matchers.equalTo; @@ -62,40 +69,96 @@ public void testRectangleShape() throws IOException { assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY) / 2))); // box-query touches bottom-left corner - assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), minX, minY))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), + minY - randomIntBetween(1, 180), minX, minY)); // box-query touches bottom-right corner - assertTrue(reader.intersects(Extent.fromPoints(maxX, minY - randomIntBetween(1, 10), maxX + randomIntBetween(1, 10), minY))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), + maxX + randomIntBetween(1, 180), minY)); // box-query touches top-right corner - assertTrue(reader.intersects(Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), + maxY + randomIntBetween(1, 180))); // box-query touches top-left corner - assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), maxY, minX, maxY + randomIntBetween(1, 10)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, + maxY + randomIntBetween(1, 180))); // box-query fully-enclosed inside rectangle - assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, - (3 * maxY + minY) / 4))); + assertRelation(GeoRelation.QUERY_INSIDE,reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); // box-query fully-contains poly - assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), minY - randomIntBetween(1, 10), - maxX + randomIntBetween(1, 10), maxY + randomIntBetween(1, 10)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), + minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); // box-query half-in-half-out-right - assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), - (3 * maxY + minY) / 4))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); // box-query half-in-half-out-left - assertTrue(reader.intersects(Extent.fromPoints(minX - randomIntBetween(1, 10), (3 * minY + maxY) / 4, (3 * maxX + minX) / 4, - (3 * maxY + minY) / 4))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, + (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); // box-query half-in-half-out-top - assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, maxX + randomIntBetween(1, 10), - maxY + randomIntBetween(1, 10)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + maxX + randomIntBetween(1, 1000), maxY + randomIntBetween(1, 1000))); // box-query half-in-half-out-bottom - assertTrue(reader.intersects(Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 10), - maxX + randomIntBetween(1, 10), (3 * maxY + minY) / 4))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); // box-query outside to the right - assertFalse(reader.intersects(Extent.fromPoints(maxX + randomIntBetween(1, 4), minY, maxX + randomIntBetween(5, 10), maxY))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, + maxX + randomIntBetween(1001, 2000), maxY)); // box-query outside to the left - assertFalse(reader.intersects(Extent.fromPoints(maxX - randomIntBetween(5, 10), minY, minX - randomIntBetween(1, 4), maxY))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, + minX - randomIntBetween(1, 1000), maxY)); // box-query outside to the top - assertFalse(reader.intersects(Extent.fromPoints(minX, maxY + randomIntBetween(1, 4), maxX, maxY + randomIntBetween(5, 10)))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, + maxY + randomIntBetween(1001, 2000))); // box-query outside to the bottom - assertFalse(reader.intersects(Extent.fromPoints(minX, minY - randomIntBetween(5, 10), maxX, minY - randomIntBetween(1, 4)))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, + minY - randomIntBetween(1, 1000))); + } + } + + public void testSimplePolygon() throws IOException { + for (int iter = 0; iter < 1000; iter++) { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "name"); + ShapeBuilder builder = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON); + Polygon geo = (Polygon) builder.buildGeometry(); + Geometry geometry = indexer.prepareForIndexing(geo); + Polygon testPolygon; + if (geometry instanceof Polygon) { + testPolygon = (Polygon) geometry; + } else if (geometry instanceof MultiPolygon) { + testPolygon = ((MultiPolygon) geometry).get(0); + } else { + throw new IllegalStateException("not a polygon"); + } + builder = LegacyGeoShapeQueryProcessor.geometryToShapeBuilder(testPolygon); + org.locationtech.spatial4j.shape.Rectangle box = builder.buildS4J().getBoundingBox(); + int minXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMinX()); + int minYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMinY()); + int maxXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMaxX()); + int maxYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMaxY()); + + double[] x = testPolygon.getPolygon().getLons(); + double[] y = testPolygon.getPolygon().getLats(); + + EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE, true); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); + Extent actualExtent = reader.getExtent(); + assertThat(actualExtent.minX(), equalTo(minXBox)); + assertThat(actualExtent.maxX(), equalTo(maxXBox)); + assertThat(actualExtent.minY(), equalTo(minYBox)); + assertThat(actualExtent.maxY(), equalTo(maxYBox)); + // polygon fully contained within box + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox)); + // relate + if (maxYBox - 1 >= minYBox) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1)); + } + if (maxXBox -1 >= minXBox) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox)); + } + // does not cross + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10)); } } @@ -107,10 +170,10 @@ public void testPacManPolygon() throws Exception { // test cell crossing poly GeometryTreeReader reader = geometryTreeReader(new Polygon(new LinearRing(py, px), Collections.emptyList()), TestCoordinateEncoder.INSTANCE); - assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); - assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); - assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); - assertTrue(reader.intersects(Extent.fromPoints(-5, -6, 2, -2))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(-5, -6, 2, -2)); } // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon @@ -120,12 +183,12 @@ public void testPolygonWithHole() throws Exception { GeometryTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); - assertFalse(reader.intersects(Extent.fromPoints(6, -6, 6, -6))); // in the hole - assertTrue(reader.intersects(Extent.fromPoints(25, -25, 25, -25))); // on the mainland - assertFalse(reader.intersects(Extent.fromPoints(51, 51, 52, 52))); // outside of mainland - assertTrue(reader.intersects(Extent.fromPoints(-60, -60, 60, 60))); // enclosing us completely - assertTrue(reader.intersects(Extent.fromPoints(49, 49, 51, 51))); // overlapping the mainland - assertTrue(reader.intersects(Extent.fromPoints(9, 9, 11, 11))); // overlapping the hole + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(6, -6, 6, -6)); // in the hole + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(25, -25, 25, -25)); // on the mainland + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(51, 51, 52, 52)); // outside of mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-60, -60, 60, 60)); // enclosing us completely + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(49, 49, 51, 51)); // overlapping the mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(9, 9, 11, 11)); // overlapping the hole } public void testCombPolygon() throws Exception { @@ -138,9 +201,9 @@ public void testCombPolygon() throws Exception { Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); GeometryTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); // test cell crossing poly - assertTrue(reader.intersects(Extent.fromPoints(5, 10, 5, 10))); - assertFalse(reader.intersects(Extent.fromPoints(15, 10, 15, 10))); - assertFalse(reader.intersects(Extent.fromPoints(25, 10, 25, 10))); + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(5, 10, 5, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(15, 10, 15, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(25, 10, 25, 10)); } public void testPacManClosedLineString() throws Exception { @@ -150,10 +213,10 @@ public void testPacManClosedLineString() throws Exception { // test cell crossing poly GeometryTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); - assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); - assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); - assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); - assertFalse(reader.intersects(Extent.fromPoints(-5, -6, 2, -2))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); } public void testPacManLineString() throws Exception { @@ -163,10 +226,10 @@ public void testPacManLineString() throws Exception { // test cell crossing poly GeometryTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); - assertTrue(reader.intersects(Extent.fromPoints(2, -1, 11, 1))); - assertTrue(reader.intersects(Extent.fromPoints(-12, -12, 12, 12))); - assertTrue(reader.intersects(Extent.fromPoints(-2, -1, 2, 0))); - assertFalse(reader.intersects(Extent.fromPoints(-5, -6, 2, -2))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); } public void testPacManPoints() throws Exception { @@ -193,7 +256,7 @@ public void testPacManPoints() throws Exception { // test cell crossing poly GeometryTreeReader reader = geometryTreeReader(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); - assertTrue(reader.intersects(Extent.fromPoints(xMin, yMin, xMax, yMax))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(xMin, yMin, xMax, yMax)); } public void testRandomMultiLineIntersections() throws IOException { @@ -207,25 +270,25 @@ public void testRandomMultiLineIntersections() throws IOException { for (Line line : geometry) { // extent that intersects edges - assertTrue(reader.intersects(bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize)); // extent that fully encloses a line in the MultiLine Extent lineExtent = geometryTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); - assertTrue(reader.intersects(lineExtent)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, lineExtent); if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { - assertTrue(reader.intersects(Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, - lineExtent.maxX() + 1, lineExtent.maxY() + 1))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, + lineExtent.maxX() + 1, lineExtent.maxY() + 1)); } } // extent that fully encloses the MultiLine - assertTrue(reader.intersects(reader.getExtent())); + assertRelation(GeoRelation.QUERY_CROSSES, reader, reader.getExtent()); if (readerExtent.minX() != Integer.MIN_VALUE && readerExtent.maxX() != Integer.MAX_VALUE && readerExtent.minY() != Integer.MIN_VALUE && readerExtent.maxY() != Integer.MAX_VALUE) { - assertTrue(reader.intersects(Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, - readerExtent.maxX() + 1, readerExtent.maxY() + 1))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, + readerExtent.maxX() + 1, readerExtent.maxY() + 1)); } } @@ -267,7 +330,7 @@ private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { return geometryTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) - .intersects(bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize)); + .relate(bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize)) == GeoRelation.QUERY_CROSSES; } private static Geometry randomGeometryTreeGeometry() { diff --git a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java index ca035b03d4587..01747a624d5f6 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.nio.ByteBuffer; +import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; import static org.hamcrest.Matchers.equalTo; public class Point2DTests extends ESTestCase { @@ -40,13 +41,17 @@ public void testOnePoint() throws IOException { Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); assertThat(reader.getExtent(), equalTo(Extent.fromPoint(x, y))); assertThat(reader.getExtent(), equalTo(reader.getExtent())); - assertTrue(reader.intersects(Extent.fromPoint(x, y))); - assertTrue(reader.intersects(Extent.fromPoints(x, y, x + randomIntBetween(1, 10), y + randomIntBetween(1, 10)))); - assertTrue(reader.intersects(Extent.fromPoints(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), x, y))); - assertTrue(reader.intersects(Extent.fromPoints(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), - x + randomIntBetween(1, 10), y + randomIntBetween(1, 10)))); - assertFalse(reader.intersects(Extent.fromPoints(x - randomIntBetween(10, 100), y - randomIntBetween(10, 100), - x - randomIntBetween(1, 10), y - randomIntBetween(1, 10)))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoint(x, y)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, + Extent.fromPoints(x, y, x + randomIntBetween(1, 10), y + randomIntBetween(1, 10))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, + Extent.fromPoints(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), x, y)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, + Extent.fromPoints(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), + x + randomIntBetween(1, 10), y + randomIntBetween(1, 10))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, + Extent.fromPoints(x - randomIntBetween(10, 100), y - randomIntBetween(10, 100), + x - randomIntBetween(1, 10), y - randomIntBetween(1, 10))); } public void testPoints() throws IOException { @@ -70,9 +75,11 @@ public void testPoints() throws IOException { writer.writeTo(output); output.close(); Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); + // tests calling getExtent() and relate() multiple times to make sure deserialization is not affected assertThat(reader.getExtent(), equalTo(reader.getExtent())); assertThat(reader.getExtent(), equalTo(writer.getExtent())); - assertTrue(reader.intersects(extent)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, extent); + assertRelation(GeoRelation.QUERY_CROSSES, reader, extent); } } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java index 82d1d69a49eb2..4a423a57b10c3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -22,14 +22,22 @@ import org.elasticsearch.common.geo.GeometryTreeReader; import org.elasticsearch.common.geo.GeometryTreeWriter; import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPolygon; import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.fielddata.MultiGeoValues; +import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.test.ESTestCase; +import java.util.Arrays; + +import static org.elasticsearch.common.geo.GeoTestUtils.geometryTreeReader; import static org.hamcrest.Matchers.equalTo; -// TODO(talevy): more tests public class GeoGridTilerTests extends ESTestCase { private static final GeoGridTiler.GeoTileGridTiler GEOTILE = GeoGridTiler.GeoTileGridTiler.INSTANCE; private static final GeoGridTiler.GeoHashGridTiler GEOHASH = GeoGridTiler.GeoHashGridTiler.INSTANCE; @@ -71,6 +79,64 @@ public void testGeoTile() throws Exception { } } + public void testGeoTileSetValuesBruteAndRecursiveMultiline() throws Exception { + int precision = randomIntBetween(0, 10); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + MultiLine geometry = GeometryTestUtils.randomMultiLine(false); + geometry = (MultiLine) indexer.prepareForIndexing(geometry); + GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + int upperBound = (int) GEOTILE.getBoundingTileCount(value, precision); + long[] recursiveValues = new long[upperBound]; + long[] bruteForceValues = new long[upperBound]; + int recursiveCount = GEOTILE.setValues(recursiveValues, value, precision); + int bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision); + Arrays.sort(recursiveValues); + Arrays.sort(bruteForceValues); + assertThat(recursiveCount, equalTo(bruteForceCount)); + assertArrayEquals(recursiveValues, bruteForceValues); + } + + public void testGeoTileSetValuesBruteAndRecursivePolygon() throws Exception { + int precision = randomIntBetween(0, 10); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry geometry = GeometryTestUtils.randomPolygon(false); + geometry = indexer.prepareForIndexing(geometry); + // TODO: support multipolygons. for now just extract first polygon + if (geometry.type() == ShapeType.MULTIPOLYGON) { + geometry = ((MultiPolygon) geometry).get(0); + } + GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + int upperBound = (int) GEOTILE.getBoundingTileCount(value, precision); + long[] recursiveValues = new long[upperBound]; + long[] bruteForceValues = new long[upperBound]; + int recursiveCount = GEOTILE.setValues(recursiveValues, value, precision); + int bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision); + Arrays.sort(recursiveValues); + Arrays.sort(bruteForceValues); + assertThat(recursiveCount, equalTo(bruteForceCount)); + assertArrayEquals(recursiveValues, bruteForceValues); + } + + public void testGeoTileSetValuesBruteAndRecursivePoints() throws Exception { + int precision = randomIntBetween(0, 10); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry geometry = randomBoolean() ? GeometryTestUtils.randomPoint(false) : GeometryTestUtils.randomMultiPoint(false); + geometry = indexer.prepareForIndexing(geometry); + GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + int upperBound = (int) GEOTILE.getBoundingTileCount(value, precision); + long[] recursiveValues = new long[upperBound]; + long[] bruteForceValues = new long[upperBound]; + int recursiveCount = GEOTILE.setValues(recursiveValues, value, precision); + int bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision); + Arrays.sort(recursiveValues); + Arrays.sort(bruteForceValues); + assertThat(recursiveCount, equalTo(bruteForceCount)); + assertArrayEquals(recursiveValues, bruteForceValues); + } + public void testGeoHash() throws Exception { double x = randomDouble(); double y = randomDouble(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index fc5cf6cb910bd..41046ca2dcd34 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; @@ -28,8 +29,10 @@ import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.keyToGeoPoint; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.longEncode; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.stringEncode; +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class GeoTileUtilsTests extends ESTestCase { @@ -231,4 +234,23 @@ public void testSingularityAtPoles() { String clippedTileIndex = stringEncode(longEncode(lon, clippedLat, zoom)); assertEquals(tileIndex, clippedTileIndex); } + + public void testPointToTile() { + int zoom = randomIntBetween(0, MAX_ZOOM); + int tiles = 1 << zoom; + int xTile = randomIntBetween(0, zoom); + int yTile = randomIntBetween(0, zoom); + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, zoom); + // check corners + assertThat(GeoTileUtils.getXTile(rectangle.getMinX(), tiles), equalTo(xTile)); + assertThat(GeoTileUtils.getXTile(rectangle.getMaxX(), tiles), equalTo(Math.min(tiles - 1, xTile + 1))); + assertThat(GeoTileUtils.getYTile(rectangle.getMaxY(), tiles), anyOf(equalTo(yTile - 1), equalTo(yTile))); + assertThat(GeoTileUtils.getYTile(rectangle.getMinY(), tiles), anyOf(equalTo(yTile + 1), equalTo(yTile))); + // check point inside + double x = randomDoubleBetween(rectangle.getMinX(), rectangle.getMaxX(), false); + double y = randomDoubleBetween(rectangle.getMinY(), rectangle.getMaxY(), false); + assertThat(GeoTileUtils.getXTile(x, tiles), equalTo(xTile)); + assertThat(GeoTileUtils.getYTile(y, tiles), equalTo(yTile)); + + } } From ab3296c834e2df14bf301882b42be5ed16cf8cf3 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 22 Nov 2019 07:35:33 -0800 Subject: [PATCH 33/62] use sloppy math in geotile-grid utils (#49350) this commit updates atan, sinh usage to ESSloppyMath, and sin usage to SloppyMath's cos. relates #37206. --- .../aggregations/bucket/geogrid/GeoTileUtils.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 0dbdba6c23fff..03013931f054f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.apache.lucene.util.SloppyMath; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.util.ESSloppyMath; @@ -42,6 +43,8 @@ */ public final class GeoTileUtils { + private static final Double PI_DIV_2 = Math.PI / 2; + private GeoTileUtils() {} /** @@ -125,7 +128,7 @@ static int getXTile(double longitude, long tiles) { * @param tiles the number of tiles per column for a pre-determined zoom-level */ static int getYTile(double latitude, long tiles) { - double latSin = Math.sin(Math.toRadians(normalizeLat(latitude))); + double latSin = SloppyMath.cos(PI_DIV_2 - Math.toRadians(normalizeLat(latitude))); int yTile = (int) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); if (yTile < 0) { @@ -151,7 +154,7 @@ public static long longEncode(double longitude, double latitude, int precision) long xTile = (long) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); - double latSin = Math.sin(Math.toRadians(normalizeLat(latitude))); + double latSin = SloppyMath.cos(PI_DIV_2 - (Math.toRadians(normalizeLat(latitude)))); long yTile = (long) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); // Edge values may generate invalid values, and need to be clipped. @@ -253,10 +256,9 @@ static Rectangle toBoundingBox(int xTile, int yTile, int precision) { final double tiles = validateZXY(precision, xTile, yTile); final double minN = Math.PI - (2.0 * Math.PI * (yTile + 1)) / tiles; final double maxN = Math.PI - (2.0 * Math.PI * (yTile)) / tiles; - final double minY = Math.toDegrees(Math.atan(Math.sinh(minN))); + final double minY = Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(minN))); final double minX = ((xTile) / tiles * 360.0) - 180; - - final double maxY = Math.toDegrees(Math.atan(Math.sinh(maxN))); + final double maxY = Math.toDegrees(ESSloppyMath.atan(ESSloppyMath.sinh(maxN))); final double maxX = ((xTile + 1) / tiles * 360.0) - 180; return new Rectangle(minX, maxX, maxY, minY); From 5d9b86bf4fd9d6c5bd615583fb17cfae7299380f Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Fri, 22 Nov 2019 14:47:14 -0800 Subject: [PATCH 34/62] Introduce variable encoding to EdgeTreeReader (#49349) This PR modifies the EdgeTree in the [geoshape-doc-values initiative](#37206) to encode the points in a variable fashion. It also adds caching to reduce the number of new Edge objects created and reduce the number of deserializations needed when an aggregation queries the shape multiple times like it does in geogrid aggregations The modifications include: - delta encoding of edge's coordinates using delta-encoding based on maxX, maxY of the Extent - remove Edge object construction and in-line all the deserialization of the edge contents within each method after these changes, two aspects of the GeometryTree feel like TODOs - reduce serialized size of Extent and simplify the `checkExtent` logic - compress Point2D tree --- .../common/geo/EdgeTreeReader.java | 177 ++++++++---------- .../common/geo/EdgeTreeWriter.java | 70 ++++--- .../common/geo/PolygonTreeWriter.java | 9 +- .../common/geo/EdgeTreeTests.java | 2 +- 4 files changed, 135 insertions(+), 123 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index 46a2bb49f4332..a6292d8d51b41 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -30,22 +30,28 @@ * serialized with the {@link EdgeTreeWriter} */ public class EdgeTreeReader implements ShapeTreeReader { - private final ByteBufferStreamInput input; - private final int startPosition; - private final boolean hasArea; private static final Optional OPTIONAL_FALSE = Optional.of(false); private static final Optional OPTIONAL_TRUE = Optional.of(true); private static final Optional OPTIONAL_EMPTY = Optional.empty(); + private final ByteBufferStreamInput input; + private final int startPosition; + private final boolean hasArea; + private Extent treeExtent; + public EdgeTreeReader(ByteBufferStreamInput input, boolean hasArea) throws IOException { this.startPosition = input.position(); this.input = input; this.hasArea = hasArea; + this.treeExtent = null; } public Extent getExtent() throws IOException { - resetInputPosition(); - return new Extent(input); + if (treeExtent == null) { + resetInputPosition(); + treeExtent = new Extent(input); + } + return treeExtent; } /** @@ -75,76 +81,69 @@ static Optional checkExtent(Extent treeExtent, Extent extent) throws IO } boolean containsBottomLeft(Extent extent) throws IOException { - resetInputPosition(); - - Optional extentCheck = checkExtent(new Extent(input), extent); + Optional extentCheck = checkExtent(getExtent(), extent); if (extentCheck.isPresent()) { return extentCheck.get(); } - return containsBottomLeft(readRoot(input.position()), extent); + resetToRootEdge(); + if (input.readBoolean()) { /* has edges */ + return containsBottomLeft(input.position(), extent); + } + return false; } public boolean crosses(Extent extent) throws IOException { resetInputPosition(); - - Optional extentCheck = checkExtent(new Extent(input), extent); + Optional extentCheck = checkExtent(getExtent(), extent); if (extentCheck.isPresent()) { return extentCheck.get(); } - return crosses(readRoot(input.position()), extent); - } - - private Edge readRoot(int position) throws IOException { - input.position(position); - if (input.readBoolean()) { - return readEdge(input.position()); + resetToRootEdge(); + if (input.readBoolean()) { /* has edges */ + return crosses(input.position(), extent); } - return null; - } - - private Edge readEdge(int position) throws IOException { - input.position(position); - int minY = input.readInt(); - int maxY = input.readInt(); - int x1 = input.readInt(); - int y1 = input.readInt(); - int x2 = input.readInt(); - int y2 = input.readInt(); - int rightOffset = input.readInt(); - return new Edge(input.position(), x1, y1, x2, y2, minY, maxY, rightOffset); - } - - - Edge readLeft(Edge root) throws IOException { - return readEdge(root.streamOffset); - } - - Edge readRight(Edge root) throws IOException { - return readEdge(root.streamOffset + root.rightOffset); + return false; } /** * Returns true if the bottom-left point of the rectangle query is contained within the * tree's edges. */ - private boolean containsBottomLeft(Edge root, Extent extent) throws IOException { + private boolean containsBottomLeft(int edgePosition, Extent extent) throws IOException { + // start read edge from bytes + input.position(edgePosition); + int maxY = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int minY = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int x1 = Math.toIntExact(treeExtent.maxX() - input.readVLong()); + int x2 = Math.toIntExact(treeExtent.maxX() - input.readVLong()); + int y1 = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int y2 = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int rightOffset = input.readVInt(); + if (rightOffset == 1) { + rightOffset = 0; + } else if (rightOffset == 0) { + rightOffset = -1; + } + int streamOffset = input.position(); + // end read edge from bytes + boolean res = false; - if (root.maxY >= extent.minY()) { + if (maxY >= extent.minY()) { // is bbox-query contained within linearRing // cast infinite ray to the right from bottom-left of bbox-query to see if it intersects edge - if (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, extent.minX(), extent.minY(), Integer.MAX_VALUE, + if (lineCrossesLineWithBoundary(x1, y1, x2, y2, extent.minX(), extent.minY(), Integer.MAX_VALUE, extent.minY())) { res = true; } - if (root.rightOffset > 0) { /* has left node */ - res ^= containsBottomLeft(readLeft(root), extent); + if (rightOffset > 0) { /* has left node */ + res ^= containsBottomLeft(streamOffset, extent); } - if (root.rightOffset >= 0 && extent.maxY() >= root.minY) { /* no right node if rightOffset == -1 */ - res ^= containsBottomLeft(readRight(root), extent); + if (rightOffset >= 0 && extent.maxY() >= minY) { /* no right node if rightOffset == -1 */ + res ^= containsBottomLeft(streamOffset + rightOffset, extent); } } return res; @@ -153,44 +152,56 @@ private boolean containsBottomLeft(Edge root, Extent extent) throws IOException /** * Returns true if the box crosses any edge in this edge subtree * */ - private boolean crosses(Edge root, Extent extent) throws IOException { - // we just have to cross one edge to answer the question, so we descend the tree and return when we do. - if (root.maxY >= extent.minY()) { + private boolean crosses(int edgePosition, Extent extent) throws IOException { + // start read edge from bytes + input.position(edgePosition); + int maxY = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int minY = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int x1 = Math.toIntExact(treeExtent.maxX() - input.readVLong()); + int x2 = Math.toIntExact(treeExtent.maxX() - input.readVLong()); + int y1 = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int y2 = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int rightOffset = input.readVInt(); + if (rightOffset == 1) { + rightOffset = 0; + } else if (rightOffset == 0) { + rightOffset = -1; + } + int streamOffset = input.position(); + // end read edge from bytes - double a1x = root.x1; - double a1y = root.y1; - double b1x = root.x2; - double b1y = root.y2; - boolean outside = (a1y < extent.minY() && b1y < extent.minY()) || - (a1y > extent.maxY() && b1y > extent.maxY()) || - (a1x < extent.minX() && b1x < extent.minX()) || - (a1x > extent.maxX() && b1x > extent.maxX()); + // we just have to cross one edge to answer the question, so we descend the tree and return when we do. + if (maxY >= extent.minY()) { + boolean outside = (y1 < extent.minY() && y2 < extent.minY()) || + (y1 > extent.maxY() && y2 > extent.maxY()) || + (x1 < extent.minX() && x2 < extent.minX()) || + (x1 > extent.maxX() && x2 > extent.maxX()); // does rectangle's edges intersect or reside inside polygon's edge - if (outside == false && (lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + if (outside == false && (lineCrossesLineWithBoundary(x1, y1, x2, y2, extent.minX(), extent.minY(), extent.maxX(), extent.minY()) || - lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + lineCrossesLineWithBoundary(x1, y1, x2, y2, extent.maxX(), extent.minY(), extent.maxX(), extent.maxY()) || - lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + lineCrossesLineWithBoundary(x1, y1, x2, y2, extent.maxX(), extent.maxY(), extent.minX(), extent.maxY()) || - lineCrossesLineWithBoundary(root.x1, root.y1, root.x2, root.y2, + lineCrossesLineWithBoundary(x1, y1, x2, y2, extent.minX(), extent.maxY(), extent.minX(), extent.minY()))) { return true; } // does this edge fully reside within the rectangle's area - if (extent.minX() <= Math.min(root.x1, root.x2) && extent.minY() <= Math.min(root.y1, root.y2) - && extent.maxX() >= Math.max(root.x1, root.x2) && extent.maxY() >= Math.max(root.y1, root.y2)) { + if (extent.minX() <= Math.min(x1, x2) && extent.minY() <= Math.min(y1, y2) + && extent.maxX() >= Math.max(x1, x2) && extent.maxY() >= Math.max(y1, y2)) { return true; } /* has left node */ - if (root.rightOffset > 0 && crosses(readLeft(root), extent)) { + if (rightOffset > 0 && crosses(streamOffset, extent)) { return true; } /* no right node if rightOffset == -1 */ - if (root.rightOffset >= 0 && extent.maxY() >= root.minY && crosses(readRight(root), extent)) { + if (rightOffset >= 0 && extent.maxY() >= minY && crosses(streamOffset + rightOffset, extent)) { return true; } } @@ -201,37 +212,7 @@ private void resetInputPosition() throws IOException { input.position(startPosition); } - private static final class Edge { - final int streamOffset; - final int x1; - final int y1; - final int x2; - final int y2; - final int minY; - final int maxY; - final int rightOffset; - - /** - * Object representing an edge node read from bytes - * - * @param streamOffset offset in byte-reference where edge terminates - * @param x1 x-coordinate of first point in segment - * @param y1 y-coordinate of first point in segment - * @param x2 x-coordinate of second point in segment - * @param y2 y-coordinate of second point in segment - * @param minY minimum y-coordinate in this edge-node's tree - * @param maxY maximum y-coordinate in this edge-node's tree - * @param rightOffset the start offset in the byte-reference of the right edge-node - */ - Edge(int streamOffset, int x1, int y1, int x2, int y2, int minY, int maxY, int rightOffset) { - this.streamOffset = streamOffset; - this.x1 = x1; - this.y1 = y1; - this.x2 = x2; - this.y2 = y2; - this.minY = minY; - this.maxY = maxY; - this.rightOffset = rightOffset; - } + private void resetToRootEdge() throws IOException { + input.position(startPosition + Extent.WRITEABLE_SIZE_IN_BYTES); // skip extent } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java index 3b11d00833cfc..e897ce40732b8 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java @@ -18,8 +18,8 @@ */ package org.elasticsearch.common.geo; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.geometry.ShapeType; import java.io.IOException; @@ -32,11 +32,6 @@ */ public class EdgeTreeWriter extends ShapeTreeWriter { - /** - * | minY | maxY | x1 | y1 | x2 | y2 | right_offset | - */ - static final int EDGE_SIZE_IN_BYTES = 28; - private final Extent extent; private final int numShapes; private final CentroidCalculator centroidCalculator; @@ -145,7 +140,12 @@ public CentroidCalculator getCentroidCalculator() { @Override public void writeTo(StreamOutput out) throws IOException { extent.writeTo(out); - out.writeOptionalWriteable(tree); + if (tree != null) { + out.writeBoolean(true); + tree.writeTo(out, new BytesStreamOutput(), extent); + } else { + out.writeBoolean(false); + } } private static Edge createTree(List edges, int low, int high) { @@ -175,7 +175,7 @@ private static Edge createTree(List edges, int low, int high) { /** * Object representing an in-memory edge-tree to be serialized */ - static class Edge implements Comparable, Writeable { + static class Edge implements Comparable { final int x1; final int y1; final int x2; @@ -204,28 +204,56 @@ public int compareTo(Edge other) { return ret; } - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeInt(minY); - out.writeInt(maxY); - out.writeInt(x1); - out.writeInt(y1); - out.writeInt(x2); - out.writeInt(y2); + private int writeEdgeContent(StreamOutput out, Extent extent) throws IOException { + long startPosition = out.position(); + out.writeVLong((long) extent.maxY() - maxY); + out.writeVLong((long) extent.maxY() - minY); + out.writeVLong((long) extent.maxX() - x1); + out.writeVLong((long) extent.maxX() - x2); + out.writeVLong((long) extent.maxY() - y1); + out.writeVLong((long) extent.maxY() - y2); + return Math.toIntExact(out.position() - startPosition); + } + + private void writeTo(StreamOutput out, BytesStreamOutput scratchBuffer, Extent extent) throws IOException { + writeEdgeContent(out, extent); + // left node is next node, write offset of right node + if (left != null) { + out.writeVInt(left.size(scratchBuffer, extent)); + } else if (right == null){ + out.writeVInt(0); + } else { + out.writeVInt(1); + } + if (left != null) { + left.writeTo(out, scratchBuffer, extent); + } + if (right != null) { + right.writeTo(out, scratchBuffer, extent); + } + } + + private int size(BytesStreamOutput scratchBuffer, Extent extent) throws IOException { + int size = writeEdgeContent(scratchBuffer, extent); + scratchBuffer.reset(); // left node is next node, write offset of right node if (left != null) { - out.writeInt(left.size * EDGE_SIZE_IN_BYTES); + int leftSize = left.size(scratchBuffer, extent); + scratchBuffer.reset(); + scratchBuffer.writeVInt(leftSize); } else if (right == null){ - out.writeInt(-1); + scratchBuffer.writeVInt(0); } else { - out.writeInt(0); + scratchBuffer.writeVInt(1); } + size += scratchBuffer.size(); if (left != null) { - left.writeTo(out); + size += left.size(scratchBuffer, extent); } if (right != null) { - right.writeTo(out); + size += right.size(scratchBuffer, extent); } + return size; } } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java index d114ae40f84c7..1a48821afcafc 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.common.geo; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; @@ -55,11 +56,13 @@ public CentroidCalculator getCentroidCalculator() { @Override public void writeTo(StreamOutput out) throws IOException { // calculate size of outerShell's tree to make it easy to jump to the holes tree quickly when querying - int size = outerShell.tree.size * EdgeTreeWriter.EDGE_SIZE_IN_BYTES + Extent.WRITEABLE_SIZE_IN_BYTES + 1; - out.writeVInt(size); + BytesStreamOutput scratchBuffer = new BytesStreamOutput(); + outerShell.writeTo(scratchBuffer); + int outerShellSize = scratchBuffer.size(); + out.writeVInt(outerShellSize); long startPosition = out.position(); outerShell.writeTo(out); - assert out.position() == size + startPosition; + assert out.position() == outerShellSize + startPosition; out.writeOptionalWriteable(holes); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java index 42bb45cc9ba7a..5901fa0228d25 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java @@ -145,7 +145,7 @@ public void testSimplePolygon() throws IOException { if (maxYBox - 1 >= minYBox) { assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1)); } - if (maxXBox -1 >= minXBox) { + if (maxXBox - 1 >= minXBox) { assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox)); } // does not cross From ffb3d27d2c9c6fa360532bf5e1af3e43659f820d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 27 Nov 2019 07:34:18 -1000 Subject: [PATCH 35/62] Add support for geometrycollections to GeometryTreeReader and Writer (#49608) Adds support geometry collections to GeometryTreeReader and GeometryTreeWriter. Relates #37206 --- .../common/geo/GeometryTreeReader.java | 35 ++++++++++--- .../common/geo/GeometryTreeWriter.java | 51 ++++++++++++++----- .../index/fielddata/MultiGeoValues.java | 2 +- .../common/geo/GeometryTreeTests.java | 28 +++++++--- 4 files changed, 87 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 5cb609be59f39..36c0710de1090 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -34,42 +34,50 @@ */ public class GeometryTreeReader implements ShapeTreeReader { - private final int extentOffset = 8; + private static final int EXTENT_OFFSET = 8; + private final int startPosition; private final ByteBufferStreamInput input; private final CoordinateEncoder coordinateEncoder; public GeometryTreeReader(BytesRef bytesRef, CoordinateEncoder coordinateEncoder) { this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + this.startPosition = 0; + this.coordinateEncoder = coordinateEncoder; + } + + private GeometryTreeReader(ByteBufferStreamInput input, CoordinateEncoder coordinateEncoder) throws IOException { + this.input = input; + startPosition = input.position(); this.coordinateEncoder = coordinateEncoder; } public double getCentroidX() throws IOException { - input.position(0); + input.position(startPosition); return coordinateEncoder.decodeX(input.readInt()); } public double getCentroidY() throws IOException { - input.position(4); + input.position(startPosition + 4); return coordinateEncoder.decodeY(input.readInt()); } @Override public Extent getExtent() throws IOException { - input.position(extentOffset); + input.position(startPosition + EXTENT_OFFSET); Extent extent = input.readOptionalWriteable(Extent::new); if (extent != null) { return extent; } assert input.readVInt() == 1; ShapeType shapeType = input.readEnum(ShapeType.class); - ShapeTreeReader reader = getReader(shapeType, input); + ShapeTreeReader reader = getReader(shapeType, coordinateEncoder, input); return reader.getExtent(); } @Override public GeoRelation relate(Extent extent) throws IOException { GeoRelation relation = GeoRelation.QUERY_DISJOINT; - input.position(extentOffset); + input.position(startPosition + EXTENT_OFFSET); boolean hasExtent = input.readBoolean(); if (hasExtent) { Optional extentCheck = EdgeTreeReader.checkExtent(new Extent(input), extent); @@ -79,9 +87,17 @@ public GeoRelation relate(Extent extent) throws IOException { } int numTrees = input.readVInt(); + int nextPosition = input.position(); for (int i = 0; i < numTrees; i++) { + if (numTrees > 1) { + if (i > 0) { + input.position(nextPosition); + } + int pos = input.readVInt(); + nextPosition = input.position() + pos; + } ShapeType shapeType = input.readEnum(ShapeType.class); - ShapeTreeReader reader = getReader(shapeType, input); + ShapeTreeReader reader = getReader(shapeType, coordinateEncoder, input); GeoRelation shapeRelation = reader.relate(extent); if (GeoRelation.QUERY_CROSSES == shapeRelation || (GeoRelation.QUERY_DISJOINT == shapeRelation && GeoRelation.QUERY_INSIDE == relation) @@ -95,7 +111,8 @@ public GeoRelation relate(Extent extent) throws IOException { return relation; } - private static ShapeTreeReader getReader(ShapeType shapeType, ByteBufferStreamInput input) throws IOException { + private static ShapeTreeReader getReader(ShapeType shapeType, CoordinateEncoder coordinateEncoder, ByteBufferStreamInput input) + throws IOException { switch (shapeType) { case POLYGON: return new PolygonTreeReader(input); @@ -105,6 +122,8 @@ private static ShapeTreeReader getReader(ShapeType shapeType, ByteBufferStreamIn case LINESTRING: case MULTILINESTRING: return new EdgeTreeReader(input, false); + case GEOMETRYCOLLECTION: + return new GeometryTreeReader(input, coordinateEncoder); default: throw new UnsupportedOperationException("unsupported shape type [" + shapeType + "]"); } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java index 61bda1d8c44ed..08222031516a2 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java @@ -18,8 +18,9 @@ */ package org.elasticsearch.common.geo; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; @@ -32,6 +33,7 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.ShapeType; import java.io.IOException; import java.util.ArrayList; @@ -43,7 +45,7 @@ * appropriate tree structure for each type of * {@link Geometry} into a byte array. */ -public class GeometryTreeWriter implements Writeable { +public class GeometryTreeWriter extends ShapeTreeWriter { private final GeometryTreeBuilder builder; private final CoordinateEncoder coordinateEncoder; @@ -53,29 +55,56 @@ public GeometryTreeWriter(Geometry geometry, CoordinateEncoder coordinateEncoder this.coordinateEncoder = coordinateEncoder; this.centroidCalculator = new CentroidCalculator(); builder = new GeometryTreeBuilder(coordinateEncoder); - geometry.visit(builder); + if (geometry.type() == ShapeType.GEOMETRYCOLLECTION) { + for (Geometry shape : (GeometryCollection) geometry) { + shape.visit(builder); + } + } else { + geometry.visit(builder); + } } - public Extent extent() { + @Override + public Extent getExtent() { return new Extent(builder.top, builder.bottom, builder.negLeft, builder.negRight, builder.posLeft, builder.posRight); } + @Override + public ShapeType getShapeType() { + return ShapeType.GEOMETRYCOLLECTION; + } + + @Override + public CentroidCalculator getCentroidCalculator() { + return centroidCalculator; + } + @Override public void writeTo(StreamOutput out) throws IOException { // only write a geometry extent for the tree if the tree // contains multiple sub-shapes - boolean prependExtent = builder.shapeWriters.size() > 1; + boolean multiShape = builder.shapeWriters.size() > 1; Extent extent = null; out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); - if (prependExtent) { + if (multiShape) { extent = new Extent(builder.top, builder.bottom, builder.negLeft, builder.negRight, builder.posLeft, builder.posRight); } out.writeOptionalWriteable(extent); out.writeVInt(builder.shapeWriters.size()); - for (ShapeTreeWriter writer : builder.shapeWriters) { - out.writeEnum(writer.getShapeType()); - writer.writeTo(out); + if (multiShape) { + for (ShapeTreeWriter writer : builder.shapeWriters) { + try(BytesStreamOutput bytesStream = new BytesStreamOutput()) { + bytesStream.writeEnum(writer.getShapeType()); + writer.writeTo(bytesStream); + BytesReference bytes = bytesStream.bytes(); + out.writeVInt(bytes.length()); + bytes.writeTo(out); + } + } + } else { + out.writeEnum(builder.shapeWriters.get(0).getShapeType()); + builder.shapeWriters.get(0).writeTo(out); } } @@ -110,9 +139,7 @@ private void addWriter(ShapeTreeWriter writer) { @Override public Void visit(GeometryCollection collection) { - for (Geometry geometry : collection) { - geometry.visit(this); - } + addWriter(new GeometryTreeWriter(collection, coordinateEncoder)); return null; } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 387555b34656c..d18865ee656b5 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -185,7 +185,7 @@ public static GeoShapeValue missing(String missing) { try { Geometry geometry = MISSING_GEOMETRY_PARSER.fromWKT(missing); GeometryTreeWriter writer = new GeometryTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); - return new GeoShapeValue(writer.extent()); + return new GeoShapeValue(writer.getExtent()); } catch (IOException | ParseException e) { throw new IllegalArgumentException("Can't apply missing value [" + missing + "]", e); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 02c1f031ad41e..c9d18f98f2ca0 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; import org.elasticsearch.geometry.Line; import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.MultiLine; @@ -33,12 +34,12 @@ import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor; -import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomShapeGenerator; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -306,10 +307,6 @@ public void testRandomGeometryIntersection() throws IOException { GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); Geometry preparedGeometry = indexer.prepareForIndexing(geometry); - // TODO: support multi-polygons - assumeFalse("polygon crosses dateline", - ShapeType.POLYGON == geometry.type() && ShapeType.MULTIPOLYGON == preparedGeometry.type()); - for (int i = 0; i < testPointCount; i++) { int cur = i; intersects[cur] = fold(preparedGeometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); @@ -329,21 +326,36 @@ private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) } private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { - return geometryTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) - .relate(bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize)) == GeoRelation.QUERY_CROSSES; + GeoRelation relation = geometryTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) + .relate(bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize)); + return relation == GeoRelation.QUERY_CROSSES || relation == GeoRelation.QUERY_INSIDE; } private static Geometry randomGeometryTreeGeometry() { + return randomGeometryTreeGeometry(0); + } + + private static Geometry randomGeometryTreeGeometry(int level) { @SuppressWarnings("unchecked") Function geometry = ESTestCase.randomFrom( GeometryTestUtils::randomLine, GeometryTestUtils::randomPoint, GeometryTestUtils::randomPolygon, GeometryTestUtils::randomMultiLine, - GeometryTestUtils::randomMultiPoint + GeometryTestUtils::randomMultiPoint, + level < 3 ? (b) -> randomGeometryTreeCollection(level + 1) : GeometryTestUtils::randomPoint // don't build too deep ); return geometry.apply(false); } + private static Geometry randomGeometryTreeCollection(int level) { + int size = ESTestCase.randomIntBetween(1, 10); + List shapes = new ArrayList<>(); + for (int i = 0; i < size; i++) { + shapes.add(randomGeometryTreeGeometry(level)); + } + return new GeometryCollection<>(shapes); + } + private GeometryTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { GeometryTreeWriter writer = new GeometryTreeWriter(geometry, encoder); BytesStreamOutput output = new BytesStreamOutput(); From b39f7127cfbb2bac8ebf3adf150d31df45cd5b60 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 28 Nov 2019 15:17:38 +0100 Subject: [PATCH 36/62] Merge branch 'master' into updateGeoshapeBranch --- .../elasticsearch/gradle/BuildPlugin.groovy | 3 + buildSrc/version.properties | 2 +- .../client/MLRequestConverters.java | 35 ++ .../client/MachineLearningClient.java | 88 ++++ .../client/ml/DeleteTrainedModelRequest.java | 64 +++ .../ml/GetTrainedModelsStatsRequest.java | 103 +++++ .../ml/GetTrainedModelsStatsResponse.java | 86 ++++ .../ml/inference/TrainedModelStats.java | 123 ++++++ .../client/MLRequestConvertersTests.java | 31 +- .../client/MachineLearningIT.java | 97 ++++- .../org/elasticsearch/client/SearchIT.java | 37 +- .../org/elasticsearch/client/SecurityIT.java | 22 +- .../MlClientDocumentationIT.java | 103 +++++ .../documentation/SearchDocumentationIT.java | 18 +- .../SecurityDocumentationIT.java | 2 - .../ml/DeleteTrainedModelRequestTests.java | 39 ++ .../ml/GetTrainedModelsStatsRequestTests.java | 39 ++ .../TrainedModelDefinitionTests.java | 8 +- .../ml/inference/TrainedModelStatsTests.java | 96 +++++ .../trainedmodel/ensemble/EnsembleTests.java | 8 +- .../trainedmodel/tree/TreeTests.java | 6 +- distribution/docker/build.gradle | 8 +- .../ml/delete-trained-model.asciidoc | 36 ++ .../ml/get-trained-models-stats.asciidoc | 42 ++ .../high-level/supported-apis.asciidoc | 4 + docs/plugins/discovery-ec2.asciidoc | 2 +- .../delimited-payload-tokenfilter.asciidoc | 326 +++++++++++++- docs/reference/indices/flush.asciidoc | 8 +- docs/reference/indices/synced-flush.asciidoc | 2 +- .../apis/enrich/put-enrich-policy.asciidoc | 2 +- docs/reference/ingest/ingest-node.asciidoc | 7 +- .../ingest/processors/enrich.asciidoc | 4 +- .../ingest/processors/pipeline.asciidoc | 2 +- .../mapping/types/parent-join.asciidoc | 15 +- .../apis/dfanalyticsresources.asciidoc | 4 +- .../apis/put-dfanalytics.asciidoc | 23 +- docs/reference/monitoring/esms.asciidoc | 8 +- docs/reference/query-dsl/mlt-query.asciidoc | 3 +- docs/reference/search/profile.asciidoc | 61 +-- .../reference/setup/bootstrap-checks.asciidoc | 6 +- docs/reference/setup/install/deb.asciidoc | 3 +- docs/reference/setup/install/docker.asciidoc | 90 ++-- docs/reference/setup/install/rpm.asciidoc | 3 +- docs/reference/setup/install/targz.asciidoc | 5 - docs/reference/setup/logging-config.asciidoc | 2 +- docs/reference/sql/functions/math.asciidoc | 2 +- modules/ingest-common/build.gradle | 6 + .../test/ingest/210_pipeline_processor.yml | 96 ++++- .../expression/MoreExpressionTests.java | 38 +- .../join/aggregations/ChildrenIT.java | 2 +- .../elasticsearch/join/query/InnerHitsIT.java | 17 +- .../join/query/ParentChildTestCase.java | 2 + .../rest-api-spec/test/20_parent_join.yml | 17 +- .../percolator/PercolatorQuerySearchIT.java | 119 ++++-- .../index/rankeval/RankEvalRequestIT.java | 20 +- .../transport/NettyAllocator.java | 2 +- .../ICUCollationKeywordFieldMapperIT.java | 89 ++-- .../AnnotatedTextFieldMapper.java | 43 +- .../AnnotatedTextFieldMapperTests.java | 2 - .../AnnotatedTextFieldTypeTests.java | 44 ++ .../test/mapper_annotatedtext/10_basic.yml | 39 +- .../azure/AzureBlobContainerRetriesTests.java | 2 +- .../azure/AzureBlobStoreTests.java | 52 --- ...CloudStorageBlobContainerRetriesTests.java | 2 +- ...leCloudStorageBlobStoreContainerTests.java | 1 - ...eCloudStorageBlobStoreRepositoryTests.java | 2 +- .../gcs/GoogleCloudStorageBlobStoreTests.java | 46 -- .../hdfs/HdfsBlobStoreContainerTests.java | 4 - .../s3/S3BlobStoreContainerTests.java | 97 ++++- .../repositories/s3/S3BlobStoreTests.java | 133 ------ .../org/elasticsearch/search/CCSDuelIT.java | 6 +- .../default/20_tar_bootstrap_password.bats | 1 - .../25_package_bootstrap_password.bats | 1 - .../bats/default/30_tar_setup_passwords.bats | 1 - .../default/35_package_setup_passwords.bats | 1 - qa/os/bats/default/bootstrap_password.bash | 170 -------- qa/os/bats/default/setup_passwords.bash | 94 ----- .../packaging/test/ArchiveTests.java | 2 +- .../packaging/test/DockerTests.java | 23 + .../packaging/test/PackagingTestCase.java | 78 +++- .../packaging/test/PasswordToolsTests.java | 141 +++++++ .../packaging/test/SqlCliTests.java | 44 ++ .../packaging/util/Archives.java | 2 +- .../elasticsearch/packaging/util/Docker.java | 75 +++- .../packaging/util/FileMatcher.java | 1 + .../packaging/util/FileUtils.java | 20 + .../packaging/util/Installation.java | 72 +++- .../packaging/util/Packages.java | 2 +- .../packaging/util/ServerUtils.java | 46 +- .../test/mixed_cluster/10_basic.yml | 2 +- .../test/old_cluster/10_basic.yml | 9 +- .../test/upgraded_cluster/10_basic.yml | 3 +- .../resources/rest-api-spec/api/delete.json | 3 +- .../resources/rest-api-spec/api/exists.json | 3 +- .../rest-api-spec/api/exists_source.json | 3 +- .../main/resources/rest-api-spec/api/get.json | 3 +- .../rest-api-spec/api/get_source.json | 3 +- .../rest-api-spec/api/mtermvectors.json | 3 +- .../rest-api-spec/api/termvectors.json | 3 +- .../20_missing_field.yml | 3 +- .../test/search/90_search_after.yml | 26 +- .../get/GetFieldMappingsIndexRequest.java | 34 +- .../mapping/get/GetFieldMappingsRequest.java | 23 +- .../get/GetFieldMappingsRequestBuilder.java | 10 - .../mapping/get/GetFieldMappingsResponse.java | 84 ++-- .../mapping/get/GetMappingsResponse.java | 4 +- .../get/TransportGetFieldMappingsAction.java | 5 +- .../TransportGetFieldMappingsIndexAction.java | 36 +- .../ingest/SimulateExecutionService.java | 2 +- .../action/update/UpdateHelper.java | 9 +- .../metadata/MetaDataIndexUpgradeService.java | 2 +- .../org/elasticsearch/common/io/Streams.java | 16 + .../common/settings/ClusterSettings.java | 1 + .../org/elasticsearch/index/IndexModule.java | 8 +- .../org/elasticsearch/index/IndexService.java | 6 +- .../index/mapper/GeoPointFieldMapper.java | 2 +- .../index/mapper/IdFieldMapper.java | 14 + .../index/mapper/MapperService.java | 14 +- .../index/mapper/TextFieldMapper.java | 2 +- .../elasticsearch/indices/IndicesService.java | 21 +- .../ingest/CompoundProcessor.java | 19 +- .../ingest/ConfigurationUtils.java | 6 + .../elasticsearch/ingest/IngestDocument.java | 17 +- .../elasticsearch/ingest/IngestService.java | 6 +- .../ingest/PipelineProcessor.java | 23 +- .../ingest/TrackingResultProcessor.java | 4 +- .../blobstore/BlobStoreRepository.java | 150 ++++--- .../indices/RestGetFieldMappingAction.java | 27 +- .../aggregations/metrics/AvgAggregator.java | 5 +- .../aggregations/metrics/CompensatedSum.java | 8 + .../metrics/ExtendedStatsAggregator.java | 6 +- .../metrics/GeoCentroidAggregator.java | 6 +- .../aggregations/metrics/StatsAggregator.java | 4 +- .../aggregations/metrics/SumAggregator.java | 3 +- .../metrics/WeightedAvgAggregator.java | 53 +-- .../search/internal/ContextIndexSearcher.java | 128 ++++-- .../search/profile/query/CollectorResult.java | 2 - .../search/query/CancellableCollector.java | 53 --- .../search/query/QueryCollectorContext.java | 15 - .../search/query/QueryPhase.java | 398 +++++++++++++++--- .../search/query/TopDocsCollectorContext.java | 2 +- .../get/GetFieldMappingsResponseTests.java | 39 +- .../action/bulk/BulkWithUpdatesIT.java | 37 ++ .../fs/FsBlobStoreContainerTests.java | 39 ++ .../common/blobstore/fs/FsBlobStoreTests.java | 81 ---- .../elasticsearch/index/IndexModuleTests.java | 2 +- .../elasticsearch/index/codec/CodecTests.java | 2 +- .../mapper/FieldFilterMapperPluginTests.java | 15 +- .../mapper/GeoPointFieldMapperTests.java | 27 ++ .../index/mapper/IdFieldMapperTests.java | 31 ++ .../mapping/SimpleGetFieldMappingsIT.java | 59 ++- .../ingest/CompoundProcessorTests.java | 79 ++++ .../elasticsearch/ingest/IngestClientIT.java | 158 ++++++- .../ingest/IngestServiceTests.java | 2 +- .../ingest/PipelineProcessorTests.java | 18 +- .../ingest/TrackingResultProcessorTests.java | 14 +- .../blobstore/BlobStoreRepositoryTests.java | 30 +- .../search/SearchCancellationTests.java | 23 +- .../search/profile/query/QueryProfilerIT.java | 5 +- .../search/query/MultiMatchQueryIT.java | 35 +- .../search/query/QueryPhaseTests.java | 321 ++++++++++---- .../search/query/SearchQueryIT.java | 35 +- .../search/sort/FieldSortIT.java | 99 ++++- .../fixture/gcs/FakeOAuth2HttpHandler.java | 18 +- .../gcs/GoogleCloudStorageHttpHandler.java | 12 +- .../elasticsearch/index/MapperTestUtils.java | 2 +- .../index/engine/EngineTestCase.java | 2 +- .../index/engine/TranslogHandler.java | 2 +- .../ESBlobStoreContainerTestCase.java | 49 ++- .../repositories/ESBlobStoreTestCase.java | 86 ---- ...ESMockAPIBasedRepositoryIntegTestCase.java | 30 +- .../test/AbstractBuilderTestCase.java | 2 +- .../elasticsearch/test/TestSearchContext.java | 9 + .../xpack/core/ml/job/messages/Messages.java | 2 +- .../core/security/authc/Authentication.java | 18 + x-pack/plugin/enrich/build.gradle | 1 + .../xpack/enrich/AbstractEnrichProcessor.java | 23 +- .../xpack/enrich/EnrichPlugin.java | 2 +- .../xpack/enrich/EnrichProcessorFactory.java | 10 +- .../xpack/enrich/GeoMatchProcessor.java | 9 +- .../xpack/enrich/MatchProcessor.java | 9 +- .../xpack/enrich/BasicEnrichTests.java | 40 +- .../enrich/EnrichProcessorFactoryTests.java | 24 +- .../xpack/enrich/GeoMatchProcessorTests.java | 5 +- .../xpack/enrich/MatchProcessorTests.java | 38 +- .../process/AnalyticsResultProcessor.java | 53 ++- .../AnalyticsResultProcessorTests.java | 30 +- .../security/authc/AuthenticationService.java | 91 ++-- .../security/rest/SecurityRestFilter.java | 6 + .../DocumentAndFieldLevelSecurityTests.java | 56 +-- .../DocumentLevelSecurityTests.java | 8 +- .../FieldLevelSecurityRandomTests.java | 23 +- .../integration/KibanaUserRoleIntegTests.java | 7 +- .../authc/AuthenticationServiceTests.java | 45 +- .../security/authc/TokenServiceTests.java | 2 +- .../sql/qa/src/main/resources/agg.csv-spec | 23 + .../qa/src/main/resources/command.csv-spec | 5 +- .../src/main/resources/conditionals.csv-spec | 26 ++ .../qa/src/main/resources/docs/docs.csv-spec | 7 +- .../plugin/sql/qa/src/main/resources/logs.csv | 2 +- .../sql/qa/src/main/resources/math.csv-spec | 10 +- .../expression/function/FunctionRegistry.java | 2 +- .../predicate/conditional/CaseProcessor.java | 15 +- .../xpack/sql/planner/QueryTranslator.java | 18 +- .../sql/planner/QueryTranslatorTests.java | 16 + ...HistoryTemplateTransformMappingsTests.java | 14 +- 206 files changed, 4681 insertions(+), 1947 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteTrainedModelRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsStatsRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsStatsResponse.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelStats.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteTrainedModelRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetTrainedModelsStatsRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelStatsTests.java create mode 100644 docs/java-rest/high-level/ml/delete-trained-model.asciidoc create mode 100644 docs/java-rest/high-level/ml/get-trained-models-stats.asciidoc create mode 100644 plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java delete mode 100644 plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java delete mode 100644 plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java delete mode 100644 plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java delete mode 120000 qa/os/bats/default/20_tar_bootstrap_password.bats delete mode 120000 qa/os/bats/default/25_package_bootstrap_password.bats delete mode 120000 qa/os/bats/default/30_tar_setup_passwords.bats delete mode 120000 qa/os/bats/default/35_package_setup_passwords.bats delete mode 100644 qa/os/bats/default/bootstrap_password.bash delete mode 100644 qa/os/bats/default/setup_passwords.bash create mode 100644 qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java create mode 100644 qa/os/src/test/java/org/elasticsearch/packaging/test/SqlCliTests.java delete mode 100644 server/src/main/java/org/elasticsearch/search/query/CancellableCollector.java delete mode 100644 server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreTests.java delete mode 100644 test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 0d715cb52488b..85e58cf5ff3c7 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -720,6 +720,9 @@ class BuildPlugin implements Plugin { // TODO: remove this once ctx isn't added to update script params in 7.0 test.systemProperty 'es.scripting.update.ctx_in_params', 'false' + // TODO: remove this property in 8.0 + test.systemProperty 'es.search.rewrite_sort', 'true' + // TODO: remove this once cname is prepended to transport.publish_address by default in 8.0 test.systemProperty 'es.transport.cname_in_publish_address', 'true' diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 81e2e2c19cdda..12387d6a08440 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -30,7 +30,7 @@ joda = 2.10.4 # - x-pack/plugin/security bouncycastle = 1.61 # test dependencies -randomizedrunner = 2.7.1 +randomizedrunner = 2.7.4 junit = 4.12 httpclient = 4.5.10 httpcore = 4.4.12 diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java index 0a1a18eeb4461..e0bba0c78a120 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java @@ -29,6 +29,7 @@ import org.elasticsearch.client.RequestConverters.EndpointBuilder; import org.elasticsearch.client.core.PageParams; import org.elasticsearch.client.ml.CloseJobRequest; +import org.elasticsearch.client.ml.DeleteTrainedModelRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.DeleteCalendarEventRequest; import org.elasticsearch.client.ml.DeleteCalendarJobRequest; @@ -60,6 +61,7 @@ import org.elasticsearch.client.ml.GetOverallBucketsRequest; import org.elasticsearch.client.ml.GetRecordsRequest; import org.elasticsearch.client.ml.GetTrainedModelsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.OpenJobRequest; import org.elasticsearch.client.ml.PostCalendarEventRequest; @@ -748,6 +750,39 @@ static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) return request; } + static Request getTrainedModelsStats(GetTrainedModelsStatsRequest getTrainedModelsStatsRequest) { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_ml", "inference") + .addPathPart(Strings.collectionToCommaDelimitedString(getTrainedModelsStatsRequest.getIds())) + .addPathPart("_stats") + .build(); + RequestConverters.Params params = new RequestConverters.Params(); + if (getTrainedModelsStatsRequest.getPageParams() != null) { + PageParams pageParams = getTrainedModelsStatsRequest.getPageParams(); + if (pageParams.getFrom() != null) { + params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString()); + } + if (pageParams.getSize() != null) { + params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString()); + } + } + if (getTrainedModelsStatsRequest.getAllowNoMatch() != null) { + params.putParam(GetTrainedModelsStatsRequest.ALLOW_NO_MATCH, + Boolean.toString(getTrainedModelsStatsRequest.getAllowNoMatch())); + } + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + request.addParameters(params.asMap()); + return request; + } + + static Request deleteTrainedModel(DeleteTrainedModelRequest deleteRequest) { + String endpoint = new EndpointBuilder() + .addPathPartAsIs("_ml", "inference") + .addPathPart(deleteRequest.getId()) + .build(); + return new Request(HttpDelete.METHOD_NAME, endpoint); + } + static Request putFilter(PutFilterRequest putFilterRequest) throws IOException { String endpoint = new EndpointBuilder() .addPathPartAsIs("_ml") diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java index 468cd535c01dc..0589f5c28d865 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.ml.CloseJobRequest; import org.elasticsearch.client.ml.CloseJobResponse; +import org.elasticsearch.client.ml.DeleteTrainedModelRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse; import org.elasticsearch.client.ml.DeleteCalendarEventRequest; @@ -76,6 +77,8 @@ import org.elasticsearch.client.ml.GetRecordsResponse; import org.elasticsearch.client.ml.GetTrainedModelsRequest; import org.elasticsearch.client.ml.GetTrainedModelsResponse; +import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsStatsResponse; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.MlInfoResponse; import org.elasticsearch.client.ml.OpenJobRequest; @@ -2337,4 +2340,89 @@ public Cancellable getTrainedModelsAsync(GetTrainedModelsRequest request, Collections.emptySet()); } + /** + * Gets trained model stats + *

+ * For additional info + * see + * GET Trained Model Stats documentation + * + * @param request The {@link GetTrainedModelsStatsRequest} + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return {@link GetTrainedModelsStatsResponse} response object + */ + public GetTrainedModelsStatsResponse getTrainedModelsStats(GetTrainedModelsStatsRequest request, + RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + MLRequestConverters::getTrainedModelsStats, + options, + GetTrainedModelsStatsResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Gets trained model stats asynchronously and notifies listener upon completion + *

+ * For additional info + * see + * GET Trained Model Stats documentation + * + * @param request The {@link GetTrainedModelsStatsRequest} + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable getTrainedModelsStatsAsync(GetTrainedModelsStatsRequest request, + RequestOptions options, + ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity(request, + MLRequestConverters::getTrainedModelsStats, + options, + GetTrainedModelsStatsResponse::fromXContent, + listener, + Collections.emptySet()); + } + + /** + * Deletes the given Trained Model + *

+ * For additional info + * see + * DELETE Trained Model documentation + * + * @param request The {@link DeleteTrainedModelRequest} + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return action acknowledgement + * @throws IOException when there is a serialization issue sending the request or receiving the response + */ + public AcknowledgedResponse deleteTrainedModel(DeleteTrainedModelRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, + MLRequestConverters::deleteTrainedModel, + options, + AcknowledgedResponse::fromXContent, + Collections.emptySet()); + } + + /** + * Deletes the given Trained Model asynchronously and notifies listener upon completion + *

+ * For additional info + * see + * DELETE Trained Model documentation + * + * @param request The {@link DeleteTrainedModelRequest} + * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener Listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public Cancellable deleteTrainedModelAsync(DeleteTrainedModelRequest request, + RequestOptions options, + ActionListener listener) { + return restHighLevelClient.performRequestAsyncAndParseEntity(request, + MLRequestConverters::deleteTrainedModel, + options, + AcknowledgedResponse::fromXContent, + listener, + Collections.emptySet()); + } } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteTrainedModelRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteTrainedModelRequest.java new file mode 100644 index 0000000000000..7deefed9ab1cb --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/DeleteTrainedModelRequest.java @@ -0,0 +1,64 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.client.ValidationException; + +import java.util.Objects; +import java.util.Optional; + +/** + * Request to delete a data frame analytics config + */ +public class DeleteTrainedModelRequest implements Validatable { + + private final String id; + + public DeleteTrainedModelRequest(String id) { + this.id = id; + } + + public String getId() { + return id; + } + + @Override + public Optional validate() { + if (id == null) { + return Optional.of(ValidationException.withError("trained model id must not be null")); + } + return Optional.empty(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + DeleteTrainedModelRequest other = (DeleteTrainedModelRequest) o; + return Objects.equals(id, other.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsStatsRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsStatsRequest.java new file mode 100644 index 0000000000000..8ad597cb18d5e --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsStatsRequest.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.Validatable; +import org.elasticsearch.client.ValidationException; +import org.elasticsearch.client.core.PageParams; +import org.elasticsearch.common.Nullable; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +public class GetTrainedModelsStatsRequest implements Validatable { + + public static final String ALLOW_NO_MATCH = "allow_no_match"; + + private final List ids; + private Boolean allowNoMatch; + private PageParams pageParams; + + /** + * Helper method to create a request that will get ALL TrainedModelStats + * @return new {@link GetTrainedModelsStatsRequest} object for the id "_all" + */ + public static GetTrainedModelsStatsRequest getAllTrainedModelStatsRequest() { + return new GetTrainedModelsStatsRequest("_all"); + } + + public GetTrainedModelsStatsRequest(String... ids) { + this.ids = Arrays.asList(ids); + } + + public List getIds() { + return ids; + } + + public Boolean getAllowNoMatch() { + return allowNoMatch; + } + + /** + * Whether to ignore if a wildcard expression matches no trained models. + * + * @param allowNoMatch If this is {@code false}, then an error is returned when a wildcard (or {@code _all}) + * does not match any trained models + */ + public GetTrainedModelsStatsRequest setAllowNoMatch(boolean allowNoMatch) { + this.allowNoMatch = allowNoMatch; + return this; + } + + public PageParams getPageParams() { + return pageParams; + } + + public GetTrainedModelsStatsRequest setPageParams(@Nullable PageParams pageParams) { + this.pageParams = pageParams; + return this; + } + + @Override + public Optional validate() { + if (ids == null || ids.isEmpty()) { + return Optional.of(ValidationException.withError("trained model id must not be null")); + } + return Optional.empty(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + GetTrainedModelsStatsRequest other = (GetTrainedModelsStatsRequest) o; + return Objects.equals(ids, other.ids) + && Objects.equals(allowNoMatch, other.allowNoMatch) + && Objects.equals(pageParams, other.pageParams); + } + + @Override + public int hashCode() { + return Objects.hash(ids, allowNoMatch, pageParams); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsStatsResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsStatsResponse.java new file mode 100644 index 0000000000000..83d17e6e2be96 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/GetTrainedModelsStatsResponse.java @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml; + +import org.elasticsearch.client.ml.inference.TrainedModelStats; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.util.List; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; + +public class GetTrainedModelsStatsResponse { + + public static final ParseField TRAINED_MODEL_STATS = new ParseField("trained_model_stats"); + public static final ParseField COUNT = new ParseField("count"); + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "get_trained_model_stats", + true, + args -> new GetTrainedModelsStatsResponse((List) args[0], (Long) args[1])); + + static { + PARSER.declareObjectArray(constructorArg(), (p, c) -> TrainedModelStats.fromXContent(p), TRAINED_MODEL_STATS); + PARSER.declareLong(constructorArg(), COUNT); + } + + public static GetTrainedModelsStatsResponse fromXContent(final XContentParser parser) { + return PARSER.apply(parser, null); + } + + private final List trainedModelStats; + private final Long count; + + + public GetTrainedModelsStatsResponse(List trainedModelStats, Long count) { + this.trainedModelStats = trainedModelStats; + this.count = count; + } + + public List getTrainedModelStats() { + return trainedModelStats; + } + + /** + * @return The total count of the trained models that matched the ID pattern. + */ + public Long getCount() { + return count; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + GetTrainedModelsStatsResponse other = (GetTrainedModelsStatsResponse) o; + return Objects.equals(this.trainedModelStats, other.trainedModelStats) && Objects.equals(this.count, other.count); + } + + @Override + public int hashCode() { + return Objects.hash(trainedModelStats, count); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelStats.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelStats.java new file mode 100644 index 0000000000000..f577ccbefe64d --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/inference/TrainedModelStats.java @@ -0,0 +1,123 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.inference; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.ingest.IngestStats; + +import java.io.IOException; +import java.util.Map; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class TrainedModelStats implements ToXContentObject { + + public static final ParseField MODEL_ID = new ParseField("model_id"); + public static final ParseField PIPELINE_COUNT = new ParseField("pipeline_count"); + public static final ParseField INGEST_STATS = new ParseField("ingest"); + + private final String modelId; + private final Map ingestStats; + private final int pipelineCount; + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>( + "trained_model_stats", + true, + args -> new TrainedModelStats((String) args[0], (Map) args[1], (Integer) args[2])); + + static { + PARSER.declareString(constructorArg(), MODEL_ID); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> p.mapOrdered(), INGEST_STATS); + PARSER.declareInt(constructorArg(), PIPELINE_COUNT); + } + + public static TrainedModelStats fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + public TrainedModelStats(String modelId, Map ingestStats, int pipelineCount) { + this.modelId = modelId; + this.ingestStats = ingestStats; + this.pipelineCount = pipelineCount; + } + + /** + * The model id for which the stats apply + */ + public String getModelId() { + return modelId; + } + + /** + * Ingest level statistics. See {@link IngestStats#toXContent(XContentBuilder, Params)} for fields and format + * If there are no ingest pipelines referencing the model, then the ingest statistics could be null. + */ + @Nullable + public Map getIngestStats() { + return ingestStats; + } + + /** + * The total number of pipelines that reference the trained model + */ + public int getPipelineCount() { + return pipelineCount; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(MODEL_ID.getPreferredName(), modelId); + builder.field(PIPELINE_COUNT.getPreferredName(), pipelineCount); + if (ingestStats != null) { + builder.field(INGEST_STATS.getPreferredName(), ingestStats); + } + builder.endObject(); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(modelId, ingestStats, pipelineCount); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + TrainedModelStats other = (TrainedModelStats) obj; + return Objects.equals(this.modelId, other.modelId) + && Objects.equals(this.ingestStats, other.ingestStats) + && Objects.equals(this.pipelineCount, other.pipelineCount); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java index 39a237f40877a..cf22ba80c1624 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MLRequestConvertersTests.java @@ -25,6 +25,7 @@ import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.core.PageParams; import org.elasticsearch.client.ml.CloseJobRequest; +import org.elasticsearch.client.ml.DeleteTrainedModelRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.DeleteCalendarEventRequest; import org.elasticsearch.client.ml.DeleteCalendarJobRequest; @@ -58,6 +59,7 @@ import org.elasticsearch.client.ml.GetOverallBucketsRequest; import org.elasticsearch.client.ml.GetRecordsRequest; import org.elasticsearch.client.ml.GetTrainedModelsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.OpenJobRequest; import org.elasticsearch.client.ml.PostCalendarEventRequest; @@ -824,7 +826,6 @@ public void testGetTrainedModels() { Request request = MLRequestConverters.getTrainedModels(getRequest); assertEquals(HttpGet.METHOD_NAME, request.getMethod()); assertEquals("/_ml/inference/" + modelId1 + "," + modelId2 + "," + modelId3, request.getEndpoint()); - assertThat(request.getParameters(), allOf(hasEntry("from", "100"), hasEntry("size", "300"), hasEntry("allow_no_match", "false"))); assertThat(request.getParameters(), allOf( hasEntry("from", "100"), @@ -836,6 +837,34 @@ public void testGetTrainedModels() { assertNull(request.getEntity()); } + public void testGetTrainedModelsStats() { + String modelId1 = randomAlphaOfLength(10); + String modelId2 = randomAlphaOfLength(10); + String modelId3 = randomAlphaOfLength(10); + GetTrainedModelsStatsRequest getRequest = new GetTrainedModelsStatsRequest(modelId1, modelId2, modelId3) + .setAllowNoMatch(false) + .setPageParams(new PageParams(100, 300)); + + Request request = MLRequestConverters.getTrainedModelsStats(getRequest); + assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals("/_ml/inference/" + modelId1 + "," + modelId2 + "," + modelId3 + "/_stats", request.getEndpoint()); + assertThat(request.getParameters(), + allOf( + hasEntry("from", "100"), + hasEntry("size", "300"), + hasEntry("allow_no_match", "false") + )); + assertNull(request.getEntity()); + } + + public void testDeleteTrainedModel() { + DeleteTrainedModelRequest deleteRequest = new DeleteTrainedModelRequest(randomAlphaOfLength(10)); + Request request = MLRequestConverters.deleteTrainedModel(deleteRequest); + assertEquals(HttpDelete.METHOD_NAME, request.getMethod()); + assertEquals("/_ml/inference/" + deleteRequest.getId(), request.getEndpoint()); + assertNull(request.getEntity()); + } + public void testPutFilter() throws IOException { MlFilter filter = MlFilterTests.createRandomBuilder("foo").build(); PutFilterRequest putFilterRequest = new PutFilterRequest(filter); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 6fd56b2ff4a3e..74c7ce0a6cda0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -24,6 +24,7 @@ import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; +import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; @@ -32,6 +33,7 @@ import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.client.ml.CloseJobRequest; import org.elasticsearch.client.ml.CloseJobResponse; +import org.elasticsearch.client.ml.DeleteTrainedModelRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse; import org.elasticsearch.client.ml.DeleteCalendarEventRequest; @@ -76,6 +78,8 @@ import org.elasticsearch.client.ml.GetModelSnapshotsResponse; import org.elasticsearch.client.ml.GetTrainedModelsRequest; import org.elasticsearch.client.ml.GetTrainedModelsResponse; +import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsStatsResponse; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.MlInfoResponse; import org.elasticsearch.client.ml.OpenJobRequest; @@ -147,6 +151,8 @@ import org.elasticsearch.client.ml.inference.TrainedModelConfig; import org.elasticsearch.client.ml.inference.TrainedModelDefinition; import org.elasticsearch.client.ml.inference.TrainedModelDefinitionTests; +import org.elasticsearch.client.ml.inference.TrainedModelStats; +import org.elasticsearch.client.ml.inference.trainedmodel.TargetType; import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.DataDescription; import org.elasticsearch.client.ml.job.config.Detector; @@ -156,6 +162,7 @@ import org.elasticsearch.client.ml.job.config.MlFilter; import org.elasticsearch.client.ml.job.process.ModelSnapshot; import org.elasticsearch.client.ml.job.stats.JobStats; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.unit.ByteSizeUnit; @@ -2092,6 +2099,94 @@ public void testGetTrainedModels() throws Exception { } } + public void testGetTrainedModelsStats() throws Exception { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + String modelIdPrefix = "get-trained-model-stats-"; + int numberOfModels = 5; + for (int i = 0; i < numberOfModels; ++i) { + String modelId = modelIdPrefix + i; + putTrainedModel(modelId); + } + + String regressionPipeline = "{" + + " \"processors\": [\n" + + " {\n" + + " \"inference\": {\n" + + " \"target_field\": \"regression_value\",\n" + + " \"model_id\": \"" + modelIdPrefix + 0 + "\",\n" + + " \"inference_config\": {\"regression\": {}},\n" + + " \"field_mappings\": {\n" + + " \"col1\": \"col1\",\n" + + " \"col2\": \"col2\",\n" + + " \"col3\": \"col3\",\n" + + " \"col4\": \"col4\"\n" + + " }\n" + + " }\n" + + " }]}\n"; + + highLevelClient().ingest().putPipeline( + new PutPipelineRequest("regression-stats-pipeline", + new BytesArray(regressionPipeline.getBytes(StandardCharsets.UTF_8)), + XContentType.JSON), + RequestOptions.DEFAULT); + { + GetTrainedModelsStatsResponse getTrainedModelsStatsResponse = execute( + GetTrainedModelsStatsRequest.getAllTrainedModelStatsRequest(), + machineLearningClient::getTrainedModelsStats, machineLearningClient::getTrainedModelsStatsAsync); + assertThat(getTrainedModelsStatsResponse.getTrainedModelStats(), hasSize(numberOfModels)); + assertThat(getTrainedModelsStatsResponse.getCount(), equalTo(5L)); + assertThat(getTrainedModelsStatsResponse.getTrainedModelStats().get(0).getPipelineCount(), equalTo(1)); + assertThat(getTrainedModelsStatsResponse.getTrainedModelStats().get(1).getPipelineCount(), equalTo(0)); + } + { + GetTrainedModelsStatsResponse getTrainedModelsStatsResponse = execute( + new GetTrainedModelsStatsRequest(modelIdPrefix + 4, modelIdPrefix + 2, modelIdPrefix + 3), + machineLearningClient::getTrainedModelsStats, machineLearningClient::getTrainedModelsStatsAsync); + assertThat(getTrainedModelsStatsResponse.getTrainedModelStats(), hasSize(3)); + assertThat(getTrainedModelsStatsResponse.getCount(), equalTo(3L)); + } + { + GetTrainedModelsStatsResponse getTrainedModelsStatsResponse = execute( + new GetTrainedModelsStatsRequest(modelIdPrefix + "*").setPageParams(new PageParams(1, 2)), + machineLearningClient::getTrainedModelsStats, machineLearningClient::getTrainedModelsStatsAsync); + assertThat(getTrainedModelsStatsResponse.getTrainedModelStats(), hasSize(2)); + assertThat(getTrainedModelsStatsResponse.getCount(), equalTo(5L)); + assertThat( + getTrainedModelsStatsResponse.getTrainedModelStats() + .stream() + .map(TrainedModelStats::getModelId) + .collect(Collectors.toList()), + containsInAnyOrder(modelIdPrefix + 1, modelIdPrefix + 2)); + } + } + + public void testDeleteTrainedModel() throws Exception { + MachineLearningClient machineLearningClient = highLevelClient().machineLearning(); + String modelId = "delete-trained-model-test"; + putTrainedModel(modelId); + + GetTrainedModelsResponse getTrainedModelsResponse = execute( + new GetTrainedModelsRequest(modelId + "*").setIncludeDefinition(false).setAllowNoMatch(true), + machineLearningClient::getTrainedModels, + machineLearningClient::getTrainedModelsAsync); + + assertThat(getTrainedModelsResponse.getCount(), equalTo(1L)); + assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(1)); + + AcknowledgedResponse deleteTrainedModelResponse = execute( + new DeleteTrainedModelRequest(modelId), + machineLearningClient::deleteTrainedModel, machineLearningClient::deleteTrainedModelAsync); + assertTrue(deleteTrainedModelResponse.isAcknowledged()); + + getTrainedModelsResponse = execute( + new GetTrainedModelsRequest(modelId + "*").setIncludeDefinition(false).setAllowNoMatch(true), + machineLearningClient::getTrainedModels, + machineLearningClient::getTrainedModelsAsync); + + assertThat(getTrainedModelsResponse.getCount(), equalTo(0L)); + assertThat(getTrainedModelsResponse.getTrainedModels(), hasSize(0)); + } + public void testPutFilter() throws Exception { String filterId = "filter-job-test"; MlFilter mlFilter = MlFilter.builder(filterId) @@ -2270,7 +2365,7 @@ private void openJob(Job job) throws IOException { } private void putTrainedModel(String modelId) throws IOException { - TrainedModelDefinition definition = TrainedModelDefinitionTests.createRandomBuilder().build(); + TrainedModelDefinition definition = TrainedModelDefinitionTests.createRandomBuilder(TargetType.REGRESSION).build(); highLevelClient().index( new IndexRequest(".ml-inference-000001") .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 5618e18742267..9df9623926fc5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -114,28 +114,28 @@ public class SearchIT extends ESRestHighLevelClientTestCase { public void indexDocuments() throws IOException { { Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/_doc/1"); - doc1.setJsonEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}"); + doc1.setJsonEntity("{\"type\":\"type1\", \"id\":1, \"num\":10, \"num2\":50}"); client().performRequest(doc1); Request doc2 = new Request(HttpPut.METHOD_NAME, "/index/_doc/2"); - doc2.setJsonEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}"); + doc2.setJsonEntity("{\"type\":\"type1\", \"id\":2, \"num\":20, \"num2\":40}"); client().performRequest(doc2); Request doc3 = new Request(HttpPut.METHOD_NAME, "/index/_doc/3"); - doc3.setJsonEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}"); + doc3.setJsonEntity("{\"type\":\"type1\", \"id\":3, \"num\":50, \"num2\":35}"); client().performRequest(doc3); Request doc4 = new Request(HttpPut.METHOD_NAME, "/index/_doc/4"); - doc4.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}"); + doc4.setJsonEntity("{\"type\":\"type2\", \"id\":4, \"num\":100, \"num2\":10}"); client().performRequest(doc4); Request doc5 = new Request(HttpPut.METHOD_NAME, "/index/_doc/5"); - doc5.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}"); + doc5.setJsonEntity("{\"type\":\"type2\", \"id\":5, \"num\":100, \"num2\":10}"); client().performRequest(doc5); } { Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1"); - doc1.setJsonEntity("{\"field\":\"value1\", \"rating\": 7}"); + doc1.setJsonEntity("{\"id\":1, \"field\":\"value1\", \"rating\": 7}"); client().performRequest(doc1); Request doc2 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/2"); - doc2.setJsonEntity("{\"field\":\"value2\"}"); + doc2.setJsonEntity("{\"id\":2, \"field\":\"value2\"}"); client().performRequest(doc2); } @@ -153,19 +153,19 @@ public void indexDocuments() throws IOException { "}"); client().performRequest(create); Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3"); - doc3.setJsonEntity("{\"field\":\"value1\", \"rating\": \"good\"}"); + doc3.setJsonEntity("{\"id\":3, \"field\":\"value1\", \"rating\": \"good\"}"); client().performRequest(doc3); Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4"); - doc4.setJsonEntity("{\"field\":\"value2\"}"); + doc4.setJsonEntity("{\"id\":4, \"field\":\"value2\"}"); client().performRequest(doc4); } { Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5"); - doc5.setJsonEntity("{\"field\":\"value1\"}"); + doc5.setJsonEntity("{\"id\":5, \"field\":\"value1\"}"); client().performRequest(doc5); Request doc6 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/6"); - doc6.setJsonEntity("{\"field\":\"value2\"}"); + doc6.setJsonEntity("{\"id\":6, \"field\":\"value2\"}"); client().performRequest(doc6); } @@ -188,7 +188,7 @@ public void indexDocuments() throws IOException { "}"); client().performRequest(create); Request doc1 = new Request(HttpPut.METHOD_NAME, "/index4/_doc/1"); - doc1.setJsonEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}"); + doc1.setJsonEntity("{\"id\":1, \"field1\":\"value1\", \"field2\":\"value2\"}"); client().performRequest(doc1); Request createFilteredAlias = new Request(HttpPost.METHOD_NAME, "/_aliases"); @@ -225,7 +225,7 @@ public void testSearchNoQuery() throws IOException { assertEquals(1.0f, searchHit.getScore(), 0); assertEquals(-1L, searchHit.getVersion()); assertNotNull(searchHit.getSourceAsMap()); - assertEquals(3, searchHit.getSourceAsMap().size()); + assertEquals(4, searchHit.getSourceAsMap().size()); assertTrue(searchHit.getSourceAsMap().containsKey("type")); assertTrue(searchHit.getSourceAsMap().containsKey("num")); assertTrue(searchHit.getSourceAsMap().containsKey("num2")); @@ -249,7 +249,7 @@ public void testSearchMatchQuery() throws IOException { assertThat(searchHit.getScore(), greaterThan(0f)); assertEquals(-1L, searchHit.getVersion()); assertNotNull(searchHit.getSourceAsMap()); - assertEquals(3, searchHit.getSourceAsMap().size()); + assertEquals(4, searchHit.getSourceAsMap().size()); assertEquals("type1", searchHit.getSourceAsMap().get("type")); assertEquals(50, searchHit.getSourceAsMap().get("num2")); } @@ -705,13 +705,13 @@ public void testSearchScroll() throws Exception { public void testMultiSearch() throws Exception { MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); SearchRequest searchRequest1 = new SearchRequest("index1"); - searchRequest1.source().sort("_id", SortOrder.ASC); + searchRequest1.source().sort("id", SortOrder.ASC); multiSearchRequest.add(searchRequest1); SearchRequest searchRequest2 = new SearchRequest("index2"); - searchRequest2.source().sort("_id", SortOrder.ASC); + searchRequest2.source().sort("id", SortOrder.ASC); multiSearchRequest.add(searchRequest2); SearchRequest searchRequest3 = new SearchRequest("index3"); - searchRequest3.source().sort("_id", SortOrder.ASC); + searchRequest3.source().sort("id", SortOrder.ASC); multiSearchRequest.add(searchRequest3); MultiSearchResponse multiSearchResponse = @@ -1198,7 +1198,8 @@ public void testExplainWithFetchSource() throws IOException { assertTrue(explainResponse.hasExplanation()); assertThat(explainResponse.getExplanation().getValue(), equalTo(1.0f)); assertTrue(explainResponse.getGetResult().isExists()); - assertThat(explainResponse.getGetResult().getSource(), equalTo(Collections.singletonMap("field1", "value1"))); + assertEquals(2, explainResponse.getGetResult().getSource().size()); + assertThat(explainResponse.getGetResult().getSource().get("field1"), equalTo("value1")); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java index 8122ff17648b1..a2dc1963d08a7 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SecurityIT.java @@ -44,6 +44,7 @@ import org.elasticsearch.client.security.user.privileges.Role; import org.elasticsearch.common.CharArrays; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -71,10 +72,8 @@ public void testPutUser() throws Exception { final PutUserResponse updateUserResponse = execute(updateUserRequest, securityClient::putUser, securityClient::putUserAsync); // assert user not created assertThat(updateUserResponse.isCreated(), is(false)); - // delete user - final Request deleteUserRequest = new Request(HttpDelete.METHOD_NAME, - "/_security/user/" + putUserRequest.getUser().getUsername()); - highLevelClient().getLowLevelClient().performRequest(deleteUserRequest); + // cleanup + deleteUser(putUserRequest.getUser()); } public void testGetUser() throws Exception { @@ -91,6 +90,8 @@ public void testGetUser() throws Exception { ArrayList users = new ArrayList<>(); users.addAll(getUsersResponse.getUsers()); assertThat(users.get(0), is(putUserRequest.getUser())); + + deleteUser(putUserRequest.getUser()); } public void testAuthenticate() throws Exception { @@ -161,12 +162,17 @@ public void testPutRole() throws Exception { assertThat(deleteRoleResponse.isFound(), is(true)); } - private static User randomUser() { + private void deleteUser(User user) throws IOException { + final Request deleteUserRequest = new Request(HttpDelete.METHOD_NAME, "/_security/user/" + user.getUsername()); + highLevelClient().getLowLevelClient().performRequest(deleteUserRequest); + } + + private User randomUser() { final String username = randomAlphaOfLengthBetween(1, 4); return randomUser(username); } - private static User randomUser(String username) { + private User randomUser(String username) { final List roles = Arrays.asList(generateRandomStringArray(3, 3, false, true)); final String fullName = randomFrom(random(), null, randomAlphaOfLengthBetween(0, 3)); final String email = randomFrom(random(), null, randomAlphaOfLengthBetween(0, 3)); @@ -182,6 +188,8 @@ private static User randomUser(String username) { } else { metadata.put("string_list", Arrays.asList(generateRandomStringArray(4, 4, false, true))); } + metadata.put("test-case", getTestName()); + return new User(username, roles, metadata, fullName, email); } @@ -207,7 +215,7 @@ private static Role randomRole(String roleName) { return roleBuilder.build(); } - private static PutUserRequest randomPutUserRequest(boolean enabled) { + private PutUserRequest randomPutUserRequest(boolean enabled) { final User user = randomUser(); return randomPutUserRequest(user, enabled); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java index 92b05d3a12c4f..8c6e134822b60 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MlClientDocumentationIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.ml.CloseJobRequest; import org.elasticsearch.client.ml.CloseJobResponse; +import org.elasticsearch.client.ml.DeleteTrainedModelRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsRequest; import org.elasticsearch.client.ml.ExplainDataFrameAnalyticsResponse; import org.elasticsearch.client.ml.DeleteCalendarEventRequest; @@ -90,6 +91,8 @@ import org.elasticsearch.client.ml.GetRecordsResponse; import org.elasticsearch.client.ml.GetTrainedModelsRequest; import org.elasticsearch.client.ml.GetTrainedModelsResponse; +import org.elasticsearch.client.ml.GetTrainedModelsStatsRequest; +import org.elasticsearch.client.ml.GetTrainedModelsStatsResponse; import org.elasticsearch.client.ml.MlInfoRequest; import org.elasticsearch.client.ml.MlInfoResponse; import org.elasticsearch.client.ml.OpenJobRequest; @@ -162,6 +165,7 @@ import org.elasticsearch.client.ml.inference.TrainedModelConfig; import org.elasticsearch.client.ml.inference.TrainedModelDefinition; import org.elasticsearch.client.ml.inference.TrainedModelDefinitionTests; +import org.elasticsearch.client.ml.inference.TrainedModelStats; import org.elasticsearch.client.ml.job.config.AnalysisConfig; import org.elasticsearch.client.ml.job.config.AnalysisLimits; import org.elasticsearch.client.ml.job.config.DataDescription; @@ -3592,6 +3596,105 @@ public void onFailure(Exception e) { } } + public void testGetTrainedModelsStats() throws Exception { + putTrainedModel("my-trained-model"); + RestHighLevelClient client = highLevelClient(); + { + // tag::get-trained-models-stats-request + GetTrainedModelsStatsRequest request = + new GetTrainedModelsStatsRequest("my-trained-model") // <1> + .setPageParams(new PageParams(0, 1)) // <2> + .setAllowNoMatch(true); // <3> + // end::get-trained-models-stats-request + + // tag::get-trained-models-stats-execute + GetTrainedModelsStatsResponse response = + client.machineLearning().getTrainedModelsStats(request, RequestOptions.DEFAULT); + // end::get-trained-models-stats-execute + + // tag::get-trained-models-stats-response + List models = response.getTrainedModelStats(); + // end::get-trained-models-stats-response + + assertThat(models, hasSize(1)); + } + { + GetTrainedModelsStatsRequest request = new GetTrainedModelsStatsRequest("my-trained-model"); + + // tag::get-trained-models-stats-execute-listener + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(GetTrainedModelsStatsResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::get-trained-models-stats-execute-listener + + // Replace the empty listener by a blocking listener in test + CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::get-trained-models-stats-execute-async + client.machineLearning() + .getTrainedModelsStatsAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::get-trained-models-stats-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + + public void testDeleteTrainedModel() throws Exception { + RestHighLevelClient client = highLevelClient(); + { + putTrainedModel("my-trained-model"); + // tag::delete-trained-model-request + DeleteTrainedModelRequest request = new DeleteTrainedModelRequest("my-trained-model"); // <1> + // end::delete-trained-model-request + + // tag::delete-trained-model-execute + AcknowledgedResponse response = client.machineLearning().deleteTrainedModel(request, RequestOptions.DEFAULT); + // end::delete-trained-model-execute + + // tag::delete-trained-model-response + boolean deleted = response.isAcknowledged(); + // end::delete-trained-model-response + + assertThat(deleted, is(true)); + } + { + putTrainedModel("my-trained-model"); + DeleteTrainedModelRequest request = new DeleteTrainedModelRequest("my-trained-model"); + + // tag::delete-trained-model-execute-listener + ActionListener listener = new ActionListener<>() { + @Override + public void onResponse(AcknowledgedResponse response) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::delete-trained-model-execute-listener + + // Replace the empty listener by a blocking listener in test + CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::delete-trained-model-execute-async + client.machineLearning().deleteTrainedModelAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::delete-trained-model-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } public void testCreateFilter() throws Exception { RestHighLevelClient client = highLevelClient(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index 47ae97fe3573f..995a50508fcf3 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -169,7 +169,7 @@ public void testSearch() throws Exception { // tag::search-source-sorting sourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC)); // <1> - sourceBuilder.sort(new FieldSortBuilder("_id").order(SortOrder.ASC)); // <2> + sourceBuilder.sort(new FieldSortBuilder("id").order(SortOrder.ASC)); // <2> // end::search-source-sorting // tag::search-source-filtering-off @@ -1251,6 +1251,9 @@ private void indexSearchTestData() throws IOException { CreateIndexRequest authorsRequest = new CreateIndexRequest("authors") .mapping(XContentFactory.jsonBuilder().startObject() .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("user") .field("type", "keyword") .field("doc_values", "false") @@ -1263,6 +1266,9 @@ private void indexSearchTestData() throws IOException { CreateIndexRequest reviewersRequest = new CreateIndexRequest("contributors") .mapping(XContentFactory.jsonBuilder().startObject() .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("user") .field("type", "keyword") .field("store", "true") @@ -1274,19 +1280,19 @@ private void indexSearchTestData() throws IOException { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest("posts").id("1") - .source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?", "user", + .source(XContentType.JSON, "id", 1, "title", "In which order are my Elasticsearch queries executed?", "user", Arrays.asList("kimchy", "luca"), "innerObject", Collections.singletonMap("key", "value"))); bulkRequest.add(new IndexRequest("posts").id("2") - .source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch", "user", + .source(XContentType.JSON, "id", 2, "title", "Current status and upcoming changes in Elasticsearch", "user", Arrays.asList("kimchy", "christoph"), "innerObject", Collections.singletonMap("key", "value"))); bulkRequest.add(new IndexRequest("posts").id("3") - .source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch", "user", + .source(XContentType.JSON, "id", 3, "title", "The Future of Federated Search in Elasticsearch", "user", Arrays.asList("kimchy", "tanguy"), "innerObject", Collections.singletonMap("key", "value"))); bulkRequest.add(new IndexRequest("authors").id("1") - .source(XContentType.JSON, "user", "kimchy")); + .source(XContentType.JSON, "id", 1, "user", "kimchy")); bulkRequest.add(new IndexRequest("contributors").id("1") - .source(XContentType.JSON, "user", "tanguy")); + .source(XContentType.JSON, "id", 1, "user", "tanguy")); bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java index 91dfa2f703fbf..372be2c2d08a5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SecurityDocumentationIT.java @@ -151,7 +151,6 @@ protected Settings restAdminSettings() { .build(); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/48440") public void testGetUsers() throws Exception { final RestHighLevelClient client = highLevelClient(); String[] usernames = new String[] {"user1", "user2", "user3"}; @@ -245,7 +244,6 @@ public void onFailure(Exception e) { } } - public void testPutUser() throws Exception { RestHighLevelClient client = highLevelClient(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteTrainedModelRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteTrainedModelRequestTests.java new file mode 100644 index 0000000000000..f1ea780199004 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/DeleteTrainedModelRequestTests.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Optional; + +import static org.hamcrest.Matchers.containsString; + +public class DeleteTrainedModelRequestTests extends ESTestCase { + + public void testValidate_Ok() { + assertEquals(Optional.empty(), new DeleteTrainedModelRequest("valid-id").validate()); + assertEquals(Optional.empty(), new DeleteTrainedModelRequest("").validate()); + } + + public void testValidate_Failure() { + assertThat(new DeleteTrainedModelRequest(null).validate().get().getMessage(), + containsString("trained model id must not be null")); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetTrainedModelsStatsRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetTrainedModelsStatsRequestTests.java new file mode 100644 index 0000000000000..77f928cfc20a8 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/GetTrainedModelsStatsRequestTests.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.ml; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Optional; + +import static org.hamcrest.Matchers.containsString; + +public class GetTrainedModelsStatsRequestTests extends ESTestCase { + + public void testValidate_Ok() { + assertEquals(Optional.empty(), new GetTrainedModelsStatsRequest("valid-id").validate()); + assertEquals(Optional.empty(), new GetTrainedModelsStatsRequest("").validate()); + } + + public void testValidate_Failure() { + assertThat(new GetTrainedModelsStatsRequest(new String[0]).validate().get().getMessage(), + containsString("trained model id must not be null")); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelDefinitionTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelDefinitionTests.java index 5b52c07a844a0..714898bfefe99 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelDefinitionTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelDefinitionTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.client.ml.inference.preprocessing.FrequencyEncodingTests; import org.elasticsearch.client.ml.inference.preprocessing.OneHotEncodingTests; import org.elasticsearch.client.ml.inference.preprocessing.TargetMeanEncodingTests; +import org.elasticsearch.client.ml.inference.trainedmodel.TargetType; import org.elasticsearch.client.ml.inference.trainedmodel.ensemble.EnsembleTests; import org.elasticsearch.client.ml.inference.trainedmodel.tree.TreeTests; import org.elasticsearch.common.settings.Settings; @@ -56,6 +57,10 @@ protected Predicate getRandomFieldsExcludeFilter() { } public static TrainedModelDefinition.Builder createRandomBuilder() { + return createRandomBuilder(randomFrom(TargetType.values())); + } + + public static TrainedModelDefinition.Builder createRandomBuilder(TargetType targetType) { int numberOfProcessors = randomIntBetween(1, 10); return new TrainedModelDefinition.Builder() .setPreProcessors( @@ -65,7 +70,8 @@ public static TrainedModelDefinition.Builder createRandomBuilder() { TargetMeanEncodingTests.createRandom())) .limit(numberOfProcessors) .collect(Collectors.toList())) - .setTrainedModel(randomFrom(TreeTests.createRandom(), EnsembleTests.createRandom())); + .setTrainedModel(randomFrom(TreeTests.buildRandomTree(Collections.emptyList(), 6, targetType), + EnsembleTests.createRandom(targetType))); } @Override diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelStatsTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelStatsTests.java new file mode 100644 index 0000000000000..e51560ffad9c6 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/TrainedModelStatsTests.java @@ -0,0 +1,96 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.ml.inference; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.ingest.IngestStats; +import org.elasticsearch.test.AbstractXContentTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +public class TrainedModelStatsTests extends AbstractXContentTestCase { + + @Override + protected TrainedModelStats doParseInstance(XContentParser parser) throws IOException { + return TrainedModelStats.fromXContent(parser); + } + + @Override + protected boolean supportsUnknownFields() { + return true; + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return field -> !field.isEmpty(); + } + + @Override + protected TrainedModelStats createTestInstance() { + return new TrainedModelStats( + randomAlphaOfLength(10), + randomBoolean() ? null : randomIngestStats(), + randomInt()); + } + + private Map randomIngestStats() { + try { + List pipelineIds = Stream.generate(()-> randomAlphaOfLength(10)) + .limit(randomIntBetween(0, 10)) + .collect(Collectors.toList()); + IngestStats stats = new IngestStats( + new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()), + pipelineIds.stream().map(id -> new IngestStats.PipelineStat(id, randomStats())).collect(Collectors.toList()), + pipelineIds.stream().collect(Collectors.toMap(Function.identity(), (v) -> randomProcessorStats()))); + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.startObject(); + stats.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + return XContentHelper.convertToMap(BytesReference.bytes(builder), false, builder.contentType()).v2(); + } + } catch (IOException ex) { + fail(ex.getMessage()); + return null; + } + } + + private IngestStats.Stats randomStats(){ + return new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); + } + + private List randomProcessorStats() { + return Stream.generate(() -> randomAlphaOfLength(10)) + .limit(randomIntBetween(0, 10)) + .map(name -> new IngestStats.ProcessorStat(name, "inference", randomStats())) + .collect(Collectors.toList()); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/EnsembleTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/EnsembleTests.java index d3431fe6b8961..f2448cbf4c8bb 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/EnsembleTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/ensemble/EnsembleTests.java @@ -57,12 +57,16 @@ protected Ensemble doParseInstance(XContentParser parser) throws IOException { } public static Ensemble createRandom() { + return createRandom(randomFrom(TargetType.values())); + } + + public static Ensemble createRandom(TargetType targetType) { int numberOfFeatures = randomIntBetween(1, 10); List featureNames = Stream.generate(() -> randomAlphaOfLength(10)) .limit(numberOfFeatures) .collect(Collectors.toList()); int numberOfModels = randomIntBetween(1, 10); - List models = Stream.generate(() -> TreeTests.buildRandomTree(featureNames, 6)) + List models = Stream.generate(() -> TreeTests.buildRandomTree(featureNames, 6, targetType)) .limit(numberOfFeatures) .collect(Collectors.toList()); OutputAggregator outputAggregator = null; @@ -77,7 +81,7 @@ public static Ensemble createRandom() { return new Ensemble(featureNames, models, outputAggregator, - randomFrom(TargetType.values()), + targetType, categoryLabels); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/tree/TreeTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/tree/TreeTests.java index cb06469eaeaf1..febd1b98c2765 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/tree/TreeTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/inference/trainedmodel/tree/TreeTests.java @@ -57,10 +57,10 @@ public static Tree createRandom() { for (int i = 0; i < numberOfFeatures; i++) { featureNames.add(randomAlphaOfLength(10)); } - return buildRandomTree(featureNames, 6); + return buildRandomTree(featureNames, 6, randomFrom(TargetType.values())); } - public static Tree buildRandomTree(List featureNames, int depth) { + public static Tree buildRandomTree(List featureNames, int depth, TargetType targetType) { int numFeatures = featureNames.size(); Tree.Builder builder = Tree.builder(); builder.setFeatureNames(featureNames); @@ -88,7 +88,7 @@ public static Tree buildRandomTree(List featureNames, int depth) { categoryLabels = Arrays.asList(generateRandomStringArray(randomIntBetween(1, 10), randomIntBetween(1, 10), false, false)); } return builder.setClassificationLabels(categoryLabels) - .setTargetType(randomFrom(TargetType.values())) + .setTargetType(targetType) .build(); } diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 87af010d238ac..da940f17b0f98 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -72,7 +72,7 @@ project.ext { void addCopyDockerContextTask(final boolean oss, final boolean ubi) { task(taskName("copy", oss, ubi, "DockerContext"), type: Sync) { - expansions(oss, ubi, true).each { k, v -> + expansions(oss, ubi, true).findAll { it.key != 'build_date' }.each { k, v -> inputs.property(k, { v.toString() }) } into buildPath(oss, ubi) @@ -173,6 +173,11 @@ void addBuildDockerImage(final boolean oss, final boolean ubi) { dockerArgs.add(tag) } args dockerArgs.toArray() + File markerFile = file("build/markers/${it.name}.marker") + outputs.file(markerFile) + doLast { + markerFile.setText('', 'UTF-8') + } } assemble.dependsOn(buildDockerImageTask) BuildPlugin.requireDocker(buildDockerImageTask) @@ -209,6 +214,7 @@ subprojects { Project subProject -> final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) { executable 'docker' + outputs.file(tarFile) args "save", "-o", tarFile, diff --git a/docs/java-rest/high-level/ml/delete-trained-model.asciidoc b/docs/java-rest/high-level/ml/delete-trained-model.asciidoc new file mode 100644 index 0000000000000..0b7f54475e3e8 --- /dev/null +++ b/docs/java-rest/high-level/ml/delete-trained-model.asciidoc @@ -0,0 +1,36 @@ +-- +:api: delete-trained-model +:request: DeleteTrainedModelRequest +:response: AcknowledgedResponse +-- +[role="xpack"] +[id="{upid}-{api}"] +=== Delete Trained Model API + +experimental[] + +Deletes a previously saved Trained Model. +The API accepts a +{request}+ object and returns a +{response}+. + +[id="{upid}-{api}-request"] +==== Delete Trained Model request + +A +{request}+ requires a valid Trained Model ID. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- +<1> Constructing a new DELETE request referencing an existing Trained Model + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Response + +The returned +{response}+ object acknowledges the Trained Model deletion. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- diff --git a/docs/java-rest/high-level/ml/get-trained-models-stats.asciidoc b/docs/java-rest/high-level/ml/get-trained-models-stats.asciidoc new file mode 100644 index 0000000000000..feaf9310d0c9a --- /dev/null +++ b/docs/java-rest/high-level/ml/get-trained-models-stats.asciidoc @@ -0,0 +1,42 @@ +-- +:api: get-trained-models-stats +:request: GetTrainedModelsStatsRequest +:response: GetTrainedModelsStatsResponse +-- +[role="xpack"] +[id="{upid}-{api}"] +=== Get Trained Models Stats API + +experimental[] + +Retrieves one or more Trained Model statistics. +The API accepts a +{request}+ object and returns a +{response}+. + +[id="{upid}-{api}-request"] +==== Get Trained Models Stats request + +A +{request}+ requires either a Trained Model ID, a comma-separated list of +IDs, or the special wildcard `_all` to get stats for all Trained Models. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request] +-------------------------------------------------- +<1> Constructing a new GET request referencing an existing Trained Model +<2> Set the paging parameters +<3> Allow empty response if no Trained Models match the provided ID patterns. + If false, an error will be thrown if no Trained Models match the + ID patterns. + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== Response + +The returned +{response}+ contains the statistics +for the requested Trained Model. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response] +-------------------------------------------------- diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index d691a3ac34b09..2191e795ebbb1 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -302,6 +302,8 @@ The Java High Level REST Client supports the following Machine Learning APIs: * <<{upid}-evaluate-data-frame>> * <<{upid}-explain-data-frame-analytics>> * <<{upid}-get-trained-models>> +* <<{upid}-get-trained-models-stats>> +* <<{upid}-delete-trained-model>> * <<{upid}-put-filter>> * <<{upid}-get-filters>> * <<{upid}-update-filter>> @@ -355,6 +357,8 @@ include::ml/stop-data-frame-analytics.asciidoc[] include::ml/evaluate-data-frame.asciidoc[] include::ml/explain-data-frame-analytics.asciidoc[] include::ml/get-trained-models.asciidoc[] +include::ml/get-trained-models-stats.asciidoc[] +include::ml/delete-trained-model.asciidoc[] include::ml/put-filter.asciidoc[] include::ml/get-filters.asciidoc[] include::ml/update-filter.asciidoc[] diff --git a/docs/plugins/discovery-ec2.asciidoc b/docs/plugins/discovery-ec2.asciidoc index 27233d8e4b7ab..c1a4913170301 100644 --- a/docs/plugins/discovery-ec2.asciidoc +++ b/docs/plugins/discovery-ec2.asciidoc @@ -338,7 +338,7 @@ connections are unreliable or slow, it is not optimised for this case and its performance may suffer. If you wish to geographically distribute your data, you should provision multiple clusters and use features such as {ref}/modules-cross-cluster-search.html[cross-cluster search] and -{stack-ov}/xpack-ccr.html[cross-cluster replication]. +{ref}/xpack-ccr.html[cross-cluster replication]. ===== Other recommendations diff --git a/docs/reference/analysis/tokenfilters/delimited-payload-tokenfilter.asciidoc b/docs/reference/analysis/tokenfilters/delimited-payload-tokenfilter.asciidoc index 1cebf95033844..e0628c8086961 100644 --- a/docs/reference/analysis/tokenfilters/delimited-payload-tokenfilter.asciidoc +++ b/docs/reference/analysis/tokenfilters/delimited-payload-tokenfilter.asciidoc @@ -1,21 +1,323 @@ [[analysis-delimited-payload-tokenfilter]] -=== Delimited Payload Token Filter - -Named `delimited_payload`. Splits tokens into tokens and payload whenever a delimiter character is found. +=== Delimited payload token filter +++++ +Delimited payload +++++ [WARNING] -============================================ +==== +The older name `delimited_payload_filter` is deprecated and should not be used +with new indices. Use `delimited_payload` instead. +==== + +Separates a token stream into tokens and payloads based on a specified +delimiter. + +For example, you can use the `delimited_payload` filter with a `|` delimiter to +split `the|1 quick|2 fox|3` into the tokens `the`, `quick`, and `fox` +with respective payloads of `1`, `2`, and `3`. + +This filter uses Lucene's +https://lucene.apache.org/core/{lucene_version_path}/analyzers-common/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.html[DelimitedPayloadTokenFilter]. + +[NOTE] +.Payloads +==== +A payload is user-defined binary data associated with a token position and +stored as base64-encoded bytes. + +{es} does not store token payloads by default. To store payloads, you must: + +* Set the <> mapping parameter to + `with_positions_payloads` or `with_positions_offsets_payloads` for any field + storing payloads. +* Use an index analyzer that includes the `delimited_payload` filter + +You can view stored payloads using the <>. +==== + +[[analysis-delimited-payload-tokenfilter-analyze-ex]] +==== Example + +The following <> request uses the +`delimited_payload` filter with the default `|` delimiter to split +`the|0 brown|10 fox|5 is|0 quick|10` into tokens and payloads. + +[source,console] +-------------------------------------------------- +GET _analyze +{ + "tokenizer": "whitespace", + "filter": ["delimited_payload"], + "text": "the|0 brown|10 fox|5 is|0 quick|10" +} +-------------------------------------------------- + +The filter produces the following tokens: + +[source,text] +-------------------------------------------------- +[ the, brown, fox, is, quick ] +-------------------------------------------------- + +Note that the analyze API does not return stored payloads. For an example that +includes returned payloads, see +<>. + +///////////////////// +[source,console-result] +-------------------------------------------------- +{ + "tokens": [ + { + "token": "the", + "start_offset": 0, + "end_offset": 5, + "type": "word", + "position": 0 + }, + { + "token": "brown", + "start_offset": 6, + "end_offset": 14, + "type": "word", + "position": 1 + }, + { + "token": "fox", + "start_offset": 15, + "end_offset": 20, + "type": "word", + "position": 2 + }, + { + "token": "is", + "start_offset": 21, + "end_offset": 25, + "type": "word", + "position": 3 + }, + { + "token": "quick", + "start_offset": 26, + "end_offset": 34, + "type": "word", + "position": 4 + } + ] +} +-------------------------------------------------- +///////////////////// + +[[analysis-delimited-payload-tokenfilter-analyzer-ex]] +==== Add to an analyzer + +The following <> request uses the +`delimited-payload` filter to configure a new <>. + +[source,console] +-------------------------------------------------- +PUT delimited_payload +{ + "settings": { + "analysis": { + "analyzer": { + "whitespace_delimited_payload": { + "tokenizer": "whitespace", + "filter": [ "delimited_payload" ] + } + } + } + } +} +-------------------------------------------------- + +[[analysis-delimited-payload-tokenfilter-configure-parms]] +==== Configurable parameters + +`delimiter`:: +(Optional, string) +Character used to separate tokens from payloads. Defaults to `|`. + +`encoding`:: ++ +-- +(Optional, string) +Datatype for the stored payload. Valid values are: + +`float`::: +(Default) Float + +`identity`::: +Characters + +`int`::: +Integer +-- + +[[analysis-delimited-payload-tokenfilter-customize]] +==== Customize and add to an analyzer + +To customize the `delimited_payload` filter, duplicate it to create the basis +for a new custom token filter. You can modify the filter using its configurable +parameters. + +For example, the following <> request +uses a custom `delimited_payload` filter to configure a new +<>. The custom `delimited_payload` +filter uses the `+` delimiter to separate tokens from payloads. Payloads are +encoded as integers. + +[source,console] +-------------------------------------------------- +PUT delimited_payload_example +{ + "settings": { + "analysis": { + "analyzer": { + "whitespace_plus_delimited": { + "tokenizer": "whitespace", + "filter": [ "plus_delimited" ] + } + }, + "filter": { + "plus_delimited": { + "type": "delimited_payload", + "delimiter": "+", + "encoding": "int" + } + } + } + } +} +-------------------------------------------------- + +[[analysis-delimited-payload-tokenfilter-return-stored-payloads]] +==== Return stored payloads + +Use the <> to create an index that: + +* Includes a field that stores term vectors with payloads. +* Uses a <> with the + `delimited_payload` filter. + +[source,console] +-------------------------------------------------- +PUT text_payloads +{ + "mappings": { + "properties": { + "text": { + "type": "text", + "term_vector": "with_positions_payloads", + "analyzer": "payload_delimiter" + } + } + }, + "settings": { + "analysis": { + "analyzer": { + "payload_delimiter": { + "tokenizer": "whitespace", + "filter": [ "delimited_payload" ] + } + } + } + } +} +-------------------------------------------------- -The older name `delimited_payload_filter` is deprecated and should not be used for new indices. Use `delimited_payload` instead. +Add a document containing payloads to the index. -============================================ +[source,console] +-------------------------------------------------- +POST text_payloads/_doc/1 +{ + "text": "the|0 brown|3 fox|4 is|0 quick|10" +} +-------------------------------------------------- +// TEST[continued] -Example: "the|1 quick|2 fox|3" is split by default into tokens `the`, `quick`, and `fox` with payloads `1`, `2`, and `3` respectively. +Use the <> to return the document's tokens +and base64-encoded payloads. -Parameters: +[source,console] +-------------------------------------------------- +GET text_payloads/_termvectors/1 +{ + "fields": [ "text" ], + "payloads": true +} +-------------------------------------------------- +// TEST[continued] -`delimiter`:: - Character used for splitting the tokens. Default is `|`. +The API returns the following response: -`encoding`:: - The type of the payload. `int` for integer, `float` for float and `identity` for characters. Default is `float`. \ No newline at end of file +[source,console-result] +-------------------------------------------------- +{ + "_index": "text_payloads", + "_id": "1", + "_version": 1, + "found": true, + "took": 8, + "term_vectors": { + "text": { + "field_statistics": { + "sum_doc_freq": 5, + "doc_count": 1, + "sum_ttf": 5 + }, + "terms": { + "brown": { + "term_freq": 1, + "tokens": [ + { + "position": 1, + "payload": "QEAAAA==" + } + ] + }, + "fox": { + "term_freq": 1, + "tokens": [ + { + "position": 2, + "payload": "QIAAAA==" + } + ] + }, + "is": { + "term_freq": 1, + "tokens": [ + { + "position": 3, + "payload": "AAAAAA==" + } + ] + }, + "quick": { + "term_freq": 1, + "tokens": [ + { + "position": 4, + "payload": "QSAAAA==" + } + ] + }, + "the": { + "term_freq": 1, + "tokens": [ + { + "position": 0, + "payload": "AAAAAA==" + } + ] + } + } + } + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"took": 8/"took": "$body.took"/] diff --git a/docs/reference/indices/flush.asciidoc b/docs/reference/indices/flush.asciidoc index 627a096907ead..5729909e94f71 100644 --- a/docs/reference/indices/flush.asciidoc +++ b/docs/reference/indices/flush.asciidoc @@ -16,13 +16,13 @@ POST /twitter/_flush [[flush-api-request]] ==== {api-request-title} -`POST //flush` +`POST //_flush` -`GET //flush` +`GET //_flush` -`POST /flush` +`POST /_flush` -`GET /flush` +`GET /_flush` [[flush-api-desc]] diff --git a/docs/reference/indices/synced-flush.asciidoc b/docs/reference/indices/synced-flush.asciidoc index 35d360496fe15..99c5dd4beeadd 100644 --- a/docs/reference/indices/synced-flush.asciidoc +++ b/docs/reference/indices/synced-flush.asciidoc @@ -170,7 +170,7 @@ A replica shard failed to sync-flush. [source,console] ---- -POST /kimchy/_flush +POST /kimchy/_flush/synced ---- // TEST[s/^/PUT kimchy\n/] diff --git a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc index dfb23cf85945c..7933f5805745e 100644 --- a/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc +++ b/docs/reference/ingest/apis/enrich/put-enrich-policy.asciidoc @@ -57,7 +57,7 @@ DELETE /_enrich/policy/my-policy If you use {es} {security-features}, you must have: * `read` index privileges for any indices used -* The `enrich_user` {stack-ov}/built-in-roles.html[built-in role] +* The `enrich_user` <> // end::enrich-policy-api-prereqs[] diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 4b0016d39a837..0da0fd19e16ef 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -376,7 +376,7 @@ The `if` condition can be more then a simple equality check. The full power of the <> is available and running in the {painless}/painless-ingest-processor-context.html[ingest processor context]. -IMPORTANT: The value of ctx is read-only in `if` conditions. +IMPORTANT: The value of ctx is read-only in `if` conditions. A more complex `if` condition that drops the document (i.e. not index it) unless it has a multi-valued tag field with at least one value that contains the characters @@ -718,8 +718,9 @@ The `ignore_failure` can be set on any processor and defaults to `false`. You may want to retrieve the actual error message that was thrown by a failed processor. To do so you can access metadata fields called -`on_failure_message`, `on_failure_processor_type`, and `on_failure_processor_tag`. These fields are only accessible -from within the context of an `on_failure` block. +`on_failure_message`, `on_failure_processor_type`, `on_failure_processor_tag` and +`on_failure_pipeline` (in case an error occurred inside a pipeline processor). +These fields are only accessible from within the context of an `on_failure` block. Here is an updated version of the example that you saw earlier. But instead of setting the error message manually, the example leverages the `on_failure_message` diff --git a/docs/reference/ingest/processors/enrich.asciidoc b/docs/reference/ingest/processors/enrich.asciidoc index 3f6ad009d4ac2..f9766261f4b49 100644 --- a/docs/reference/ingest/processors/enrich.asciidoc +++ b/docs/reference/ingest/processors/enrich.asciidoc @@ -12,8 +12,8 @@ See <> section for more information about how |====== | Name | Required | Default | Description | `policy_name` | yes | - | The name of the enrich policy to use. -| `field` | yes | - | The field in the input document that matches the policies match_field used to retrieve the enrichment data. -| `target_field` | yes | - | The field that will be used for the enrichment data. +| `field` | yes | - | The field in the input document that matches the policies match_field used to retrieve the enrichment data. Supports <>. +| `target_field` | yes | - | The field that will be used for the enrichment data. Supports <>. | `ignore_missing` | no | false | If `true` and `field` does not exist, the processor quietly exits without modifying the document | `override` | no | true | If processor will update fields with pre-existing non-null-valued field. When set to `false`, such fields will not be touched. | `max_matches` | no | 1 | The maximum number of matched documents to include under the configured target field. The `target_field` will be turned into a json array if `max_matches` is higher than 1, otherwise `target_field` will become a json object. In order to avoid documents getting too large, the maximum allowed value is 128. diff --git a/docs/reference/ingest/processors/pipeline.asciidoc b/docs/reference/ingest/processors/pipeline.asciidoc index 573ab3b88d3c4..7f1ea2885e69a 100644 --- a/docs/reference/ingest/processors/pipeline.asciidoc +++ b/docs/reference/ingest/processors/pipeline.asciidoc @@ -7,7 +7,7 @@ Executes another pipeline. [options="header"] |====== | Name | Required | Default | Description -| `name` | yes | - | The name of the pipeline to execute +| `name` | yes | - | The name of the pipeline to execute. Supports <>. include::common-options.asciidoc[] |====== diff --git a/docs/reference/mapping/types/parent-join.asciidoc b/docs/reference/mapping/types/parent-join.asciidoc index 35b674bd1c61a..63056c734e1d7 100644 --- a/docs/reference/mapping/types/parent-join.asciidoc +++ b/docs/reference/mapping/types/parent-join.asciidoc @@ -16,6 +16,9 @@ PUT my_index { "mappings": { "properties": { + "my_id": { + "type": "keyword" + }, "my_join_field": { <1> "type": "join", "relations": { @@ -38,6 +41,7 @@ For instance the following example creates two `parent` documents in the `questi -------------------------------------------------- PUT my_index/_doc/1?refresh { + "my_id": "1", "text": "This is a question", "my_join_field": { "name": "question" <1> @@ -46,6 +50,7 @@ PUT my_index/_doc/1?refresh PUT my_index/_doc/2?refresh { + "my_id": "2", "text": "This is another question", "my_join_field": { "name": "question" @@ -63,12 +68,14 @@ as a shortcut instead of encapsulating it in the normal object notation: -------------------------------------------------- PUT my_index/_doc/1?refresh { + "my_id": "1", "text": "This is a question", "my_join_field": "question" <1> } PUT my_index/_doc/2?refresh { + "my_id": "2", "text": "This is another question", "my_join_field": "question" } @@ -89,6 +96,7 @@ For instance the following example shows how to index two `child` documents: -------------------------------------------------- PUT my_index/_doc/3?routing=1&refresh <1> { + "my_id": "3", "text": "This is an answer", "my_join_field": { "name": "answer", <2> @@ -98,6 +106,7 @@ PUT my_index/_doc/3?routing=1&refresh <1> PUT my_index/_doc/4?routing=1&refresh { + "my_id": "4", "text": "This is another answer", "my_join_field": { "name": "answer", @@ -159,7 +168,7 @@ GET my_index/_search "query": { "match_all": {} }, - "sort": ["_id"] + "sort": ["my_id"] } -------------------------- // TEST[continued] @@ -182,6 +191,7 @@ Will return: "_id": "1", "_score": null, "_source": { + "my_id": "1", "text": "This is a question", "my_join_field": "question" <1> }, @@ -194,6 +204,7 @@ Will return: "_id": "2", "_score": null, "_source": { + "my_id": "2", "text": "This is another question", "my_join_field": "question" <2> }, @@ -207,6 +218,7 @@ Will return: "_score": null, "_routing": "1", "_source": { + "my_id": "3", "text": "This is an answer", "my_join_field": { "name": "answer", <3> @@ -223,6 +235,7 @@ Will return: "_score": null, "_routing": "1", "_source": { + "my_id": "4", "text": "This is another answer", "my_join_field": { "name": "answer", diff --git a/docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc b/docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc index d32ce8b980653..62b5b121528a5 100644 --- a/docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc +++ b/docs/reference/ml/df-analytics/apis/dfanalyticsresources.asciidoc @@ -271,8 +271,8 @@ examining the loss on the training data. Subject to the size constraint, this operation provides an upper bound on the improvement in validation loss. A fixed number of rounds is used for optimization which depends on the number of -parameters being optimized. The optimitazion starts with random search, then -Bayesian Optimisation is performed that is targeting maximum expected +parameters being optimized. The optimization starts with random search, then +Bayesian optimization is performed that is targeting maximum expected improvement. If you override any parameters, then the optimization will calculate the value of the remaining parameters accordingly and use the value you provided for the overridden parameter. The number of rounds are reduced diff --git a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc index 531077660c9d9..159f0cb61a0c4 100644 --- a/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc +++ b/docs/reference/ml/df-analytics/apis/put-dfanalytics.asciidoc @@ -60,21 +60,22 @@ may contain documents that don't have an {olscore}. ====== {regression-cap} -{regression-cap} supports fields that are numeric, boolean, text, keyword and ip. It -is also tolerant of missing values. Fields that are supported are included in -the analysis, other fields are ignored. Documents where included fields contain -an array with two or more values are also ignored. Documents in the `dest` index -that don’t contain a results field are not included in the {reganalysis}. +{regression-cap} supports fields that are numeric, `boolean`, `text`, `keyword`, +and `ip`. It is also tolerant of missing values. Fields that are supported are +included in the analysis, other fields are ignored. Documents where included +fields contain an array with two or more values are also ignored. Documents in +the `dest` index that don’t contain a results field are not included in the + {reganalysis}. ====== {classification-cap} -{classification-cap} supports fields that are numeric, boolean, text, keyword -and ip. It is also tolerant of missing values. Fields that are supported are -included in the analysis, other fields are ignored. Documents where included -fields contain an array with two or more values are also ignored. Documents in -the `dest` index that don’t contain a results field are not included in the -{classanalysis}. +{classification-cap} supports fields that are numeric, `boolean`, `text`, +`keyword`, and `ip`. It is also tolerant of missing values. Fields that are +supported are included in the analysis, other fields are ignored. Documents +where included fields contain an array with two or more values are also ignored. +Documents in the `dest` index that don’t contain a results field are not +included in the {classanalysis}. {classanalysis-cap} can be improved by mapping ordinal variable values to a single number. For example, in case of age ranges, you can model the values as diff --git a/docs/reference/monitoring/esms.asciidoc b/docs/reference/monitoring/esms.asciidoc index b4fbcd8a65bb5..7648fe74ac70f 100644 --- a/docs/reference/monitoring/esms.asciidoc +++ b/docs/reference/monitoring/esms.asciidoc @@ -78,7 +78,7 @@ output.elasticsearch: <1> Replace `MONITORING_ELASTICSEARCH_URL` with the appropriate URL for {esms-init}, which was provided by the Elastic support team. <2> The Elastic support team creates this user in {esms-init} and grants it the -{stack-ov}/built-in-roles.html[`remote_monitoring_agent` built-in role]. +<>. <3> Replace `MONITORING_AGENT_PASSWORD` with the value provided to you by the Elastic support team. // end::metricbeat-config[] @@ -180,7 +180,7 @@ If the Elastic {security-features} are enabled, you must also provide a user ID and password so that {metricbeat} can collect metrics successfully: .. Create a user on the production cluster that has the -`remote_monitoring_collector` {ref}/built-in-roles.html[built-in role]. +`remote_monitoring_collector` <>. Alternatively, use the `remote_monitoring_user` <>. @@ -337,9 +337,9 @@ If the Elastic {security-features} are enabled, you must also provide a user ID and password so that {metricbeat} can collect metrics successfully: .. Create a user on the production cluster that has the -`remote_monitoring_collector` {ref}/built-in-roles.html[built-in role]. +`remote_monitoring_collector` <>. Alternatively, if it's available in your environment, use the -`remote_monitoring_user` {ref}/built-in-users.html[built-in user]. +`remote_monitoring_user` <>. .. Add the `username` and `password` settings to the beat module configuration file. diff --git a/docs/reference/query-dsl/mlt-query.asciidoc b/docs/reference/query-dsl/mlt-query.asciidoc index 659d0542adb41..1a440e418db64 100644 --- a/docs/reference/query-dsl/mlt-query.asciidoc +++ b/docs/reference/query-dsl/mlt-query.asciidoc @@ -198,7 +198,8 @@ input document. Defaults to `5`. `max_doc_freq`:: The maximum document frequency above which the terms will be ignored from the input document. This could be useful in order to ignore highly frequent words -such as stop words. Defaults to unbounded (`0`). +such as stop words. Defaults to unbounded (`Integer.MAX_VALUE`, which is `2^31-1` +or 2147483647). `min_word_length`:: The minimum word length below which the terms will be ignored. The old name diff --git a/docs/reference/search/profile.asciidoc b/docs/reference/search/profile.asciidoc index 0b959f87e0e84..561ed30a8cc74 100644 --- a/docs/reference/search/profile.asciidoc +++ b/docs/reference/search/profile.asciidoc @@ -153,16 +153,9 @@ The API returns the following result: "rewrite_time": 51443, "collector": [ { - "name": "CancellableCollector", - "reason": "search_cancelled", - "time_in_nanos": "304311", - "children": [ - { - "name": "SimpleTopScoreDocCollector", - "reason": "search_top_hits", - "time_in_nanos": "32273" - } - ] + "name": "SimpleTopScoreDocCollector", + "reason": "search_top_hits", + "time_in_nanos": "32273" } ] } @@ -445,16 +438,9 @@ Looking at the previous example: -------------------------------------------------- "collector": [ { - "name": "CancellableCollector", - "reason": "search_cancelled", - "time_in_nanos": "304311", - "children": [ - { - "name": "SimpleTopScoreDocCollector", - "reason": "search_top_hits", - "time_in_nanos": "32273" - } - ] + "name": "SimpleTopScoreDocCollector", + "reason": "search_top_hits", + "time_in_nanos": "32273" } ] -------------------------------------------------- @@ -657,33 +643,26 @@ The API returns the following result: "rewrite_time": 7208, "collector": [ { - "name": "CancellableCollector", - "reason": "search_cancelled", - "time_in_nanos": 2390, + "name": "MultiCollector", + "reason": "search_multi", + "time_in_nanos": 1820, "children": [ { - "name": "MultiCollector", - "reason": "search_multi", - "time_in_nanos": 1820, + "name": "FilteredCollector", + "reason": "search_post_filter", + "time_in_nanos": 7735, "children": [ { - "name": "FilteredCollector", - "reason": "search_post_filter", - "time_in_nanos": 7735, - "children": [ - { - "name": "SimpleTopScoreDocCollector", - "reason": "search_top_hits", - "time_in_nanos": 1328 - } - ] - }, - { - "name": "MultiBucketCollector: [[my_scoped_agg, my_global_agg]]", - "reason": "aggregation", - "time_in_nanos": 8273 + "name": "SimpleTopScoreDocCollector", + "reason": "search_top_hits", + "time_in_nanos": 1328 } ] + }, + { + "name": "MultiBucketCollector: [[my_scoped_agg, my_global_agg]]", + "reason": "aggregation", + "time_in_nanos": 8273 } ] } diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index 83209cbf3b853..f65526eb3612e 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -143,9 +143,9 @@ maximum size virtual memory check enforces that the Elasticsearch process has unlimited address space and is enforced only on Linux. To pass the maximum size virtual memory check, you must configure your system to allow the Elasticsearch process the ability to have unlimited -address space. This can be done via `/etc/security/limits.conf` using -the `as` setting to `unlimited` (note that you might have to increase -the limits for the `root` user too). +address space. This can be done via adding ` - as unlimited` +to `/etc/security/limits.conf`. This may require you to increase the limits +for the `root` user too. === Maximum map count check diff --git a/docs/reference/setup/install/deb.asciidoc b/docs/reference/setup/install/deb.asciidoc index 12c240b17db11..30ce4a5c0264b 100644 --- a/docs/reference/setup/install/deb.asciidoc +++ b/docs/reference/setup/install/deb.asciidoc @@ -240,7 +240,8 @@ locations for a Debian-based system: | jdk | The bundled Java Development Kit used to run Elasticsearch. Can - be overriden by setting the `JAVA_HOME` environment variable. + be overriden by setting the `JAVA_HOME` environment variable + in `/etc/default/elasticsearch`. | /usr/share/elasticsearch/jdk d| diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc index c527c5f21c3fe..add4792bd10af 100644 --- a/docs/reference/setup/install/docker.asciidoc +++ b/docs/reference/setup/install/docker.asciidoc @@ -9,11 +9,11 @@ https://www.docker.elastic.co[www.docker.elastic.co]. The source files are in https://github.com/elastic/elasticsearch/blob/{branch}/distribution/docker[Github]. -These images are free to use under the Elastic license. They contain open source -and free commercial features and access to paid commercial features. -{stack-ov}/license-management.html[Start a 30-day trial] to try out all of the -paid commercial features. See the -https://www.elastic.co/subscriptions[Subscriptions] page for information about +These images are free to use under the Elastic license. They contain open source +and free commercial features and access to paid commercial features. +{stack-ov}/license-management.html[Start a 30-day trial] to try out all of the +paid commercial features. See the +https://www.elastic.co/subscriptions[Subscriptions] page for information about Elastic license levels. ==== Pulling the image @@ -35,9 +35,9 @@ ifeval::["{release-state}"!="unreleased"] docker pull {docker-repo}:{version} -------------------------------------------- -Alternatively, you can download other Docker images that contain only features -available under the Apache 2.0 license. To download the images, go to -https://www.docker.elastic.co[www.docker.elastic.co]. +Alternatively, you can download other Docker images that contain only features +available under the Apache 2.0 license. To download the images, go to +https://www.docker.elastic.co[www.docker.elastic.co]. endif::[] @@ -52,7 +52,7 @@ endif::[] ifeval::["{release-state}"!="unreleased"] -To start a single-node {es} cluster for development or testing, specify +To start a single-node {es} cluster for development or testing, specify <> to bypass the <>: [source,sh,subs="attributes"] @@ -65,7 +65,7 @@ endif::[] [[docker-compose-file]] ==== Starting a multi-node cluster with Docker Compose -To get a three-node {es} cluster up and running in Docker, +To get a three-node {es} cluster up and running in Docker, you can use Docker Compose: . Create a `docker-compose.yml` file: @@ -84,20 +84,20 @@ include::docker-compose.yml[] -------------------------------------------- endif::[] -This sample Docker Compose file brings up a three-node {es} cluster. +This sample Docker Compose file brings up a three-node {es} cluster. Node `es01` listens on `localhost:9200` and `es02` and `es03` talk to `es01` over a Docker network. -The https://docs.docker.com/storage/volumes[Docker named volumes] -`data01`, `data02`, and `data03` store the node data directories so the data persists across restarts. +The https://docs.docker.com/storage/volumes[Docker named volumes] +`data01`, `data02`, and `data03` store the node data directories so the data persists across restarts. If they don't already exist, `docker-compose` creates them when you bring up the cluster. -- -. Make sure Docker Engine is allotted at least 4GiB of memory. +. Make sure Docker Engine is allotted at least 4GiB of memory. In Docker Desktop, you configure resource usage on the Advanced tab in Preference (macOS) or Settings (Windows). + NOTE: Docker Compose is not pre-installed with Docker on Linux. -See docs.docker.com for installation instructions: -https://docs.docker.com/compose/install[Install Compose on Linux] +See docs.docker.com for installation instructions: +https://docs.docker.com/compose/install[Install Compose on Linux] . Run `docker-compose` to bring up the cluster: + @@ -114,13 +114,13 @@ curl -X GET "localhost:9200/_cat/nodes?v&pretty" -------------------------------------------------- // NOTCONSOLE -Log messages go to the console and are handled by the configured Docker logging driver. +Log messages go to the console and are handled by the configured Docker logging driver. By default you can access logs with `docker logs`. -To stop the cluster, run `docker-compose down`. -The data in the Docker volumes is preserved and loaded +To stop the cluster, run `docker-compose down`. +The data in the Docker volumes is preserved and loaded when you restart the cluster with `docker-compose up`. -To **delete the data volumes** when you bring down the cluster, +To **delete the data volumes** when you bring down the cluster, specify the `-v` option: `docker-compose down -v`. @@ -137,7 +137,7 @@ The following requirements and recommendations apply when running {es} in Docker ===== Set `vm.max_map_count` to at least `262144` -The `vm.max_map_count` kernel setting must be set to at least `262144` for production use. +The `vm.max_map_count` kernel setting must be set to at least `262144` for production use. How you set `vm.max_map_count` depends on your platform: @@ -151,7 +151,7 @@ grep vm.max_map_count /etc/sysctl.conf vm.max_map_count=262144 -------------------------------------------- -To apply the setting on a live system, run: +To apply the setting on a live system, run: [source,sh] -------------------------------------------- @@ -196,15 +196,15 @@ sudo sysctl -w vm.max_map_count=262144 ===== Configuration files must be readable by the `elasticsearch` user By default, {es} runs inside the container as user `elasticsearch` using -uid:gid `1000:1000`. +uid:gid `1000:0`. IMPORTANT: One exception is https://docs.openshift.com/container-platform/3.6/creating_images/guidelines.html#openshift-specific-guidelines[Openshift], -which runs containers using an arbitrarily assigned user ID. +which runs containers using an arbitrarily assigned user ID. Openshift presents persistent volumes with the gid set to `0`, which works without any adjustments. If you are bind-mounting a local directory or file, it must be readable by the `elasticsearch` user. -In addition, this user must have write access to the <>. -A good strategy is to grant group access to gid `1000` or `0` for the local directory. +In addition, this user must have write access to the <>. +A good strategy is to grant group access to gid `0` for the local directory. For example, to prepare a local directory for storing data through a bind-mount: @@ -212,7 +212,7 @@ For example, to prepare a local directory for storing data through a bind-mount: -------------------------------------------- mkdir esdatadir chmod g+rwx esdatadir -chgrp 1000 esdatadir +chgrp 0 esdatadir -------------------------------------------- As a last resort, you can force the container to mutate the ownership of @@ -223,10 +223,10 @@ uid:gid `1000:0`, which provides the required read/write access to the {es} proc ===== Increase ulimits for nofile and nproc -Increased ulimits for <> and <> -must be available for the {es} containers. +Increased ulimits for <> and <> +must be available for the {es} containers. Verify the https://github.com/moby/moby/tree/ea4d1243953e6b652082305a9c3cda8656edab26/contrib/init[init system] -for the Docker daemon sets them to acceptable values. +for the Docker daemon sets them to acceptable values. To check the Docker daemon defaults for ulimits, run: @@ -246,12 +246,12 @@ For example, when using `docker run`, set: ===== Disable swapping Swapping needs to be disabled for performance and node stability. -For information about ways to do this, see <>. +For information about ways to do this, see <>. -If you opt for the `bootstrap.memory_lock: true` approach, +If you opt for the `bootstrap.memory_lock: true` approach, you also need to define the `memlock: true` ulimit in the https://docs.docker.com/engine/reference/commandline/dockerd/#default-ulimits[Docker Daemon], -or explicitly set for the container as shown in the <>. +or explicitly set for the container as shown in the <>. When using `docker run`, you can specify: -e "bootstrap.memory_lock=true" --ulimit memlock=-1:-1 @@ -260,12 +260,12 @@ When using `docker run`, you can specify: The image https://docs.docker.com/engine/reference/builder/#/expose[exposes] TCP ports 9200 and 9300. For production clusters, randomizing the -published ports with `--publish-all` is recommended, +published ports with `--publish-all` is recommended, unless you are pinning one container per host. ===== Set the heap size -Use the `ES_JAVA_OPTS` environment variable to set the heap size. +Use the `ES_JAVA_OPTS` environment variable to set the heap size. For example, to use 16GB, specify `-e ES_JAVA_OPTS="-Xms16g -Xmx16g"` with `docker run`. IMPORTANT: You must <> even if you are @@ -277,7 +277,7 @@ memory access] to the container. Pin your deployments to a specific version of the {es} Docker image. For example +docker.elastic.co/elasticsearch/elasticsearch:{version}+. -===== Always bind data volumes +===== Always bind data volumes You should use a volume bound on `/usr/share/elasticsearch/data` for the following reasons: @@ -291,7 +291,7 @@ https://docs.docker.com/engine/extend/plugins/#volume-plugins[Docker volume plug ===== Avoid using `loop-lvm` mode If you are using the devicemapper storage driver, do not use the default `loop-lvm` mode. -Configure docker-engine to use +Configure docker-engine to use https://docs.docker.com/engine/userguide/storagedriver/device-mapper-driver/#configure-docker-with-devicemapper[direct-lvm]. ===== Centralize your logs @@ -304,14 +304,14 @@ production use. [[docker-configuration-methods]] ==== Configuring {es} with Docker -When you run in Docker, the <> are loaded from +When you run in Docker, the <> are loaded from `/usr/share/elasticsearch/config/`. To use custom configuration files, you <> -over the configuration files in the image. +over the configuration files in the image. -You can set individual {es} configuration parameters using Docker environment variables. -The <> and the +You can set individual {es} configuration parameters using Docker environment variables. +The <> and the <> use this method. To use the contents of a file to set an environment variable, suffix the environment @@ -335,8 +335,8 @@ parameters as command line options. For example: docker run bin/elasticsearch -Ecluster.name=mynewclustername -------------------------------------------- -While bind-mounting your configuration files is usually the preferred method in production, -you can also <<_c_customized_image, create a custom Docker image>> +While bind-mounting your configuration files is usually the preferred method in production, +you can also <<_c_customized_image, create a custom Docker image>> that contains your configuration. [[docker-config-bind-mount]] @@ -351,7 +351,7 @@ For example, to bind-mount `custom_elasticsearch.yml` with `docker run`, specify -------------------------------------------- IMPORTANT: The container **runs {es} as user `elasticsearch` using -**uid:gid `1000:1000`**. Bind mounted host directories and files must be accessible by this user, +uid:gid `1000:0`**. Bind mounted host directories and files must be accessible by this user, and the data and log directories must be writable by this user. [[_c_customized_image]] @@ -373,7 +373,7 @@ docker build --tag=elasticsearch-custom . docker run -ti -v /usr/share/elasticsearch/data elasticsearch-custom -------------------------------------------- -Some plugins require additional security permissions. +Some plugins require additional security permissions. You must explicitly accept them either by: * Attaching a `tty` when you run the Docker image and allowing the permissions when prompted. diff --git a/docs/reference/setup/install/rpm.asciidoc b/docs/reference/setup/install/rpm.asciidoc index 1917d52c20923..4108756286eed 100644 --- a/docs/reference/setup/install/rpm.asciidoc +++ b/docs/reference/setup/install/rpm.asciidoc @@ -227,7 +227,8 @@ locations for an RPM-based system: | jdk | The bundled Java Development Kit used to run Elasticsearch. Can - be overriden by setting the `JAVA_HOME` environment variable. + be overriden by setting the `JAVA_HOME` environment variable + in `/etc/sysconfig/elasticsearch`. | /usr/share/elasticsearch/jdk d| diff --git a/docs/reference/setup/install/targz.asciidoc b/docs/reference/setup/install/targz.asciidoc index 17e678cfba0d6..d8f6cb00414f3 100644 --- a/docs/reference/setup/install/targz.asciidoc +++ b/docs/reference/setup/install/targz.asciidoc @@ -169,11 +169,6 @@ directory so that you do not delete important data later on. d| Not configured | path.repo -| script - | Location of script files. - | $ES_HOME/scripts - | path.scripts - |======================================================================= diff --git a/docs/reference/setup/logging-config.asciidoc b/docs/reference/setup/logging-config.asciidoc index d1705a887bcd5..e9a85c83f398d 100644 --- a/docs/reference/setup/logging-config.asciidoc +++ b/docs/reference/setup/logging-config.asciidoc @@ -45,7 +45,7 @@ appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB <15> -------------------------------------------------- <1> Configure the `RollingFile` appender -<2> Log to `/var/log/elasticsearch/production.json` +<2> Log to `/var/log/elasticsearch/production_server.json` <3> Use JSON layout. <4> `type_name` is a flag populating the `type` field in a `ESJsonLayout`. It can be used to distinguish different types of logs more easily when parsing them. diff --git a/docs/reference/sql/functions/math.asciidoc b/docs/reference/sql/functions/math.asciidoc index ebef8a305bcc6..8f5a24b3a1d93 100644 --- a/docs/reference/sql/functions/math.asciidoc +++ b/docs/reference/sql/functions/math.asciidoc @@ -385,7 +385,7 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[mathInlineSqrt] -------------------------------------------------- [[sql-functions-math-truncate]] -==== `TRUNCATE` +==== `TRUNCATE/TRUNC` .Synopsis: [source, sql] diff --git a/modules/ingest-common/build.gradle b/modules/ingest-common/build.gradle index ec8f8b7c3717d..6590bdc1c52ef 100644 --- a/modules/ingest-common/build.gradle +++ b/modules/ingest-common/build.gradle @@ -28,3 +28,9 @@ dependencies { compile project(':libs:elasticsearch-grok') compile project(':libs:elasticsearch-dissect') } + +testClusters.integTest { + // Needed in order to test ingest pipeline templating: + // (this is because the integTest node is not using default distribution, but only the minimal number of required modules) + module file(project(':modules:lang-mustache').tasks.bundlePlugin.archiveFile) +} diff --git a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml index 5df08b7cf90d0..76dbc180fa0e5 100644 --- a/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml +++ b/modules/ingest-common/src/test/resources/rest-api-spec/test/ingest/210_pipeline_processor.yml @@ -107,4 +107,98 @@ teardown: pipeline: "outer" body: {} - match: { error.root_cause.0.type: "ingest_processor_exception" } -- match: { error.root_cause.0.reason: "java.lang.IllegalStateException: Cycle detected for pipeline: inner" } +- match: { error.root_cause.0.reason: "java.lang.IllegalStateException: Cycle detected for pipeline: outer" } + +--- +"Test Pipeline Processor with templating": + - do: + ingest.put_pipeline: + id: "engineering-department" + body: > + { + "processors" : [ + { + "set" : { + "field": "manager", + "value": "john" + } + } + ] + } + - match: { acknowledged: true } + + - do: + ingest.put_pipeline: + id: "sales-department" + body: > + { + "processors" : [ + { + "set" : { + "field": "manager", + "value": "jan" + } + } + ] + } + - match: { acknowledged: true } + + - do: + ingest.put_pipeline: + id: "outer" + body: > + { + "processors" : [ + { + "pipeline" : { + "name": "{{org}}-department" + } + } + ] + } + - match: { acknowledged: true } + + - do: + index: + index: test + id: 1 + pipeline: "outer" + body: > + { + "org": "engineering" + } + + - do: + get: + index: test + id: 1 + - match: { _source.manager: "john" } + + - do: + index: + index: test + id: 2 + pipeline: "outer" + body: > + { + "org": "sales" + } + + - do: + get: + index: test + id: 2 + - match: { _source.manager: "jan" } + + - do: + catch: /illegal_state_exception/ + index: + index: test + id: 3 + pipeline: "outer" + body: > + { + "org": "legal" + } + - match: { error.root_cause.0.type: "ingest_processor_exception" } + - match: { error.root_cause.0.reason: "java.lang.IllegalStateException: Pipeline processor configured for non-existent pipeline [legal-department]" } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java index 8f8c30e3ec587..ca15aa01e4c59 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/MoreExpressionTests.java @@ -79,8 +79,7 @@ private SearchRequestBuilder buildRequest(String script, Object... params) { SearchRequestBuilder req = client().prepareSearch().setIndices("test"); req.setQuery(QueryBuilders.matchAllQuery()) - .addSort(SortBuilders.fieldSort("_id") - .order(SortOrder.ASC)) + .addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC).unmappedType("long")) .addScriptField("foo", new Script(ScriptType.INLINE, "expression", script, paramsMap)); return req; } @@ -147,8 +146,10 @@ public void testDateMethods() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date")); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), - client().prepareIndex("test").setId("2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); + client().prepareIndex("test").setId("1") + .setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), + client().prepareIndex("test").setId("2") + .setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); SearchResponse rsp = buildRequest("doc['date0'].getSeconds() - doc['date0'].getMinutes()").get(); assertEquals(2, rsp.getHits().getTotalHits().value); SearchHits hits = rsp.getHits(); @@ -175,8 +176,10 @@ public void testDateObjectMethods() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "date0", "type=date", "date1", "type=date")); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), - client().prepareIndex("test").setId("2").setSource("date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); + client().prepareIndex("test").setId("1") + .setSource("id", 1, "date0", "2015-04-28T04:02:07Z", "date1", "1985-09-01T23:11:01Z"), + client().prepareIndex("test").setId("2") + .setSource("id", 2, "date0", "2013-12-25T11:56:45Z", "date1", "1983-10-13T23:15:00Z")); SearchResponse rsp = buildRequest("doc['date0'].date.secondOfMinute - doc['date0'].date.minuteOfHour").get(); assertEquals(2, rsp.getHits().getTotalHits().value); SearchHits hits = rsp.getHits(); @@ -207,15 +210,18 @@ public void testMultiValueMethods() throws Exception { ensureGreen("test"); Map doc1 = new HashMap<>(); + doc1.put("id", 1); doc1.put("double0", new Double[]{5.0d, 1.0d, 1.5d}); doc1.put("double1", new Double[]{1.2d, 2.4d}); doc1.put("double2", 3.0d); Map doc2 = new HashMap<>(); + doc2.put("id", 2); doc2.put("double0", 5.0d); doc2.put("double1", 3.0d); Map doc3 = new HashMap<>(); + doc3.put("id", 3); doc3.put("double0", new Double[]{5.0d, 1.0d, 1.5d, -1.5d}); doc3.put("double1", 4.0d); @@ -319,8 +325,8 @@ public void testSparseField() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long")); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("x", 4), - client().prepareIndex("test").setId("2").setSource("y", 2)); + client().prepareIndex("test").setId("1").setSource("id", 1, "x", 4), + client().prepareIndex("test").setId("2").setSource("id", 2, "y", 2)); SearchResponse rsp = buildRequest("doc['x'] + 1").get(); ElasticsearchAssertions.assertSearchResponse(rsp); SearchHits hits = rsp.getHits(); @@ -348,9 +354,9 @@ public void testParams() throws Exception { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("x", 10), - client().prepareIndex("test").setId("2").setSource("x", 3), - client().prepareIndex("test").setId("3").setSource("x", 5)); + client().prepareIndex("test").setId("1").setSource("id", 1, "x", 10), + client().prepareIndex("test").setId("2").setSource("id", 2, "x", 3), + client().prepareIndex("test").setId("3").setSource("id", 3, "x", 5)); // a = int, b = double, c = long String script = "doc['x'] * a + b + ((c + doc['x']) > 5000000009 ? 1 : 0)"; SearchResponse rsp = buildRequest(script, "a", 2, "b", 3.5, "c", 5000000000L).get(); @@ -621,9 +627,9 @@ public void testBoolean() throws Exception { assertAcked(prepareCreate("test").addMapping("doc", xContentBuilder)); ensureGreen(); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("price", 1.0, "vip", true), - client().prepareIndex("test").setId("2").setSource("price", 2.0, "vip", false), - client().prepareIndex("test").setId("3").setSource("price", 2.0, "vip", false)); + client().prepareIndex("test").setId("1").setSource("id", 1, "price", 1.0, "vip", true), + client().prepareIndex("test").setId("2").setSource("id", 2, "price", 2.0, "vip", false), + client().prepareIndex("test").setId("3").setSource("id", 3, "price", 2.0, "vip", false)); // access .value SearchResponse rsp = buildRequest("doc['vip'].value").get(); assertSearchResponse(rsp); @@ -652,8 +658,8 @@ public void testFilterScript() throws Exception { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test").setId("1").setSource("foo", 1.0), - client().prepareIndex("test").setId("2").setSource("foo", 0.0)); + client().prepareIndex("test").setId("1").setSource("id", 1, "foo", 1.0), + client().prepareIndex("test").setId("2").setSource("id", 2, "foo", 0.0)); SearchRequestBuilder builder = buildRequest("doc['foo'].value"); Script script = new Script(ScriptType.INLINE, "expression", "doc['foo'].value", Collections.emptyMap()); builder.setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.scriptQuery(script))); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java index 33e5c86b10b3a..150c1dc76afa5 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/aggregations/ChildrenIT.java @@ -103,7 +103,7 @@ public void testParentWithMultipleBuckets() throws Exception { .setQuery(matchQuery("randomized", false)) .addAggregation( terms("category").field("category").size(10000).subAggregation( - children("to_comment", "comment").subAggregation(topHits("top_comments").sort("_id", SortOrder.ASC)) + children("to_comment", "comment").subAggregation(topHits("top_comments").sort("id", SortOrder.ASC)) ) ).get(); assertSearchResponse(searchResponse); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java index a5c56ee11be29..f6163eeca8aca 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/InnerHitsIT.java @@ -189,6 +189,9 @@ public void testSimpleParentChild() throws Exception { public void testRandomParentChild() throws Exception { assertAcked(prepareCreate("idx") .addMapping("doc", jsonBuilder().startObject().startObject("doc").startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("join_field") .field("type", "join") .startObject("relations") @@ -225,13 +228,13 @@ public void testRandomParentChild() throws Exception { BoolQueryBuilder boolQuery = new BoolQueryBuilder(); boolQuery.should(constantScoreQuery(hasChildQuery("child1", matchAllQuery(), ScoreMode.None) .innerHit(new InnerHitBuilder().setName("a") - .addSort(new FieldSortBuilder("_id").order(SortOrder.ASC)).setSize(size)))); + .addSort(new FieldSortBuilder("id").order(SortOrder.ASC)).setSize(size)))); boolQuery.should(constantScoreQuery(hasChildQuery("child2", matchAllQuery(), ScoreMode.None) .innerHit(new InnerHitBuilder().setName("b") - .addSort(new FieldSortBuilder("_id").order(SortOrder.ASC)).setSize(size)))); + .addSort(new FieldSortBuilder("id").order(SortOrder.ASC)).setSize(size)))); SearchResponse searchResponse = client().prepareSearch("idx") .setSize(numDocs) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(boolQuery) .get(); @@ -283,7 +286,7 @@ public void testInnerHitsOnHasParent() throws Exception { indexRandom(true, requests); SearchResponse response = client().prepareSearch("stack") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery( boolQuery() .must(matchQuery("body", "fail2ban")) @@ -386,7 +389,7 @@ public void testRoyals() throws Exception { hasChildQuery("baron", matchAllQuery(), ScoreMode.None) .innerHit(new InnerHitBuilder().setName("barons")), ScoreMode.None).innerHit(new InnerHitBuilder() - .addSort(SortBuilders.fieldSort("_id").order(SortOrder.ASC)) + .addSort(SortBuilders.fieldSort("id").order(SortOrder.ASC)) .setName("earls") .setSize(4)) ) @@ -440,7 +443,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { SearchResponse response = client().prepareSearch("index") .setQuery(hasChildQuery("child", matchQuery("field", "value1").queryName("_name1"), ScoreMode.None) .innerHit(new InnerHitBuilder())) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -457,7 +460,7 @@ public void testMatchesQueriesParentChildInnerHits() throws Exception { .innerHit(new InnerHitBuilder()); response = client().prepareSearch("index") .setQuery(query) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); diff --git a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java index 8436e2b8a3207..99e02aaf3d7cd 100644 --- a/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java +++ b/modules/parent-join/src/test/java/org/elasticsearch/join/query/ParentChildTestCase.java @@ -65,6 +65,7 @@ protected IndexRequestBuilder createIndexRequest(String index, String type, Stri for (int i = 0; i < fields.length; i += 2) { source.put((String) fields[i], fields[i + 1]); } + source.put("id", id); return createIndexRequest(index, type, id, parentId, source); } @@ -93,6 +94,7 @@ public static Map buildParentJoinFieldMappingFromSimplifiedDef(S } joinField.put("relations", relationMap); fields.put(joinFieldName, joinField); + fields.put("id", Collections.singletonMap("type", "keyword")); return Collections.singletonMap("properties", fields); } diff --git a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml index 9d8488f5c1451..0b65c744ec0b2 100644 --- a/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml +++ b/modules/parent-join/src/test/resources/rest-api-spec/test/20_parent_join.yml @@ -6,46 +6,47 @@ setup: mappings: properties: join_field: { "type": "join", "relations": { "parent": "child", "child": "grand_child" } } + id: { "type": "keyword" } - do: index: index: test id: 1 - body: { "join_field": { "name": "parent" } } + body: { "id", "1", "join_field": { "name": "parent" } } - do: index: index: test id: 2 - body: { "join_field": { "name": "parent" } } + body: { "id", "2", "join_field": { "name": "parent" } } - do: index: index: test id: 3 routing: 1 - body: { "join_field": { "name": "child", "parent": "1" } } + body: { "id", "3", "join_field": { "name": "child", "parent": "1" } } - do: index: index: test id: 4 routing: 1 - body: { "join_field": { "name": "child", "parent": "1" } } + body: { "id", "4", "join_field": { "name": "child", "parent": "1" } } - do: index: index: test id: 5 routing: 1 - body: { "join_field": { "name": "child", "parent": "2" } } + body: { "id", "5", "join_field": { "name": "child", "parent": "2" } } - do: index: index: test id: 6 routing: 1 - body: { "join_field": { "name": "grand_child", "parent": "5" } } + body: { "id", "6", "join_field": { "name": "grand_child", "parent": "5" } } - do: indices.refresh: {} @@ -55,7 +56,7 @@ setup: - do: search: rest_total_hits_as_int: true - body: { sort: ["join_field", "_id"] } + body: { sort: ["join_field", "id"] } - match: { hits.total: 6 } - match: { hits.hits.0._index: "test" } @@ -92,7 +93,7 @@ setup: search: rest_total_hits_as_int: true body: - sort: [ "_id" ] + sort: [ "id" ] query: parent_id: type: child diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java index 92a1802b39623..041073ebcfc3d 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchIT.java @@ -68,17 +68,23 @@ public class PercolatorQuerySearchIT extends ESIntegTestCase { public void testPercolatorQuery() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") + .addMapping("type", "id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") ); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("query", matchAllQuery()).endObject()) .get(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "2") + .field("query", matchQuery("field1", "value")).endObject()) .get(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", boolQuery() + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("query", boolQuery() .must(matchQuery("field1", "value")) .must(matchQuery("field2", "value")) ).endObject()).get(); @@ -96,7 +102,7 @@ public void testPercolatorQuery() throws Exception { logger.info("percolating doc with 1 field"); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -108,7 +114,7 @@ public void testPercolatorQuery() throws Exception { logger.info("percolating doc with 2 fields"); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -124,7 +130,7 @@ public void testPercolatorQuery() throws Exception { BytesReference.bytes(jsonBuilder().startObject().field("field1", "value").endObject()), BytesReference.bytes(jsonBuilder().startObject().field("field1", "value").field("field2", "value").endObject()) ), XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -247,22 +253,26 @@ public void testPercolatorRangeQueries() throws Exception { public void testPercolatorGeoQueries() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", "type=geo_point", "field2", "type=geo_shape", "query", "type=percolator") - ); + .addMapping("type", "id", "type=keyword", + "field1", "type=geo_point", "field2", "type=geo_shape", "query", "type=percolator")); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", - geoDistanceQuery("field1").point(52.18, 4.38).distance(50, DistanceUnit.KILOMETERS)) + .setSource(jsonBuilder().startObject() + .field("query", geoDistanceQuery("field1").point(52.18, 4.38).distance(50, DistanceUnit.KILOMETERS)) + .field("id", "1") .endObject()).get(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", - geoBoundingBoxQuery("field1").setCorners(52.3, 4.4, 52.1, 4.6)) + .setSource(jsonBuilder().startObject() + .field("query", geoBoundingBoxQuery("field1").setCorners(52.3, 4.4, 52.1, 4.6)) + .field("id", "2") .endObject()).get(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", - geoPolygonQuery("field1", Arrays.asList(new GeoPoint(52.1, 4.4), new GeoPoint(52.3, 4.5), new GeoPoint(52.1, 4.6)))) + .setSource(jsonBuilder().startObject() + .field("query", + geoPolygonQuery("field1", Arrays.asList(new GeoPoint(52.1, 4.4), new GeoPoint(52.3, 4.5), new GeoPoint(52.1, 4.6)))) + .field("id", "3") .endObject()).get(); refresh(); @@ -271,7 +281,7 @@ public void testPercolatorGeoQueries() throws Exception { .endObject()); SearchResponse response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -281,24 +291,29 @@ public void testPercolatorGeoQueries() throws Exception { public void testPercolatorQueryExistingDocument() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") + .addMapping("type", "id", "type=keyword", "field1", "type=keyword", "field2", "type=keyword", "query", "type=percolator") ); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", matchAllQuery()).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("query", matchAllQuery()).endObject()) .get(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "value")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "2") + .field("query", matchQuery("field1", "value")).endObject()) .get(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", boolQuery() + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("query", boolQuery() .must(matchQuery("field1", "value")) - .must(matchQuery("field2", "value")) - ).endObject()).get(); + .must(matchQuery("field2", "value"))).endObject()).get(); - client().prepareIndex("test").setId("4").setSource("{}", XContentType.JSON).get(); - client().prepareIndex("test").setId("5").setSource("field1", "value").get(); - client().prepareIndex("test").setId("6").setSource("field1", "value", "field2", "value").get(); + client().prepareIndex("test").setId("4").setSource("{\"id\": \"4\"}", XContentType.JSON).get(); + client().prepareIndex("test").setId("5").setSource("id", "5", "field1", "value").get(); + client().prepareIndex("test").setId("6").setSource("id", "6", "field1", "value", "field2", "value").get(); client().admin().indices().prepareRefresh().get(); logger.info("percolating empty doc"); @@ -311,7 +326,7 @@ public void testPercolatorQueryExistingDocument() throws Exception { logger.info("percolating doc with 1 field"); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", "test", "5", null, null, null)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -320,7 +335,7 @@ public void testPercolatorQueryExistingDocument() throws Exception { logger.info("percolating doc with 2 fields"); response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", "test", "6", null, null, null)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -351,15 +366,19 @@ public void testPercolatorQueryExistingDocumentSourceDisabled() throws Exception public void testPercolatorSpecificQueries() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", "type=text", "field2", "type=text", "query", "type=percolator") + .addMapping("type", "id", "type=keyword", "field1", "type=text", "field2", "type=text", "query", "type=percolator") ); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", multiMatchQuery("quick brown fox", "field1", "field2") + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("query", multiMatchQuery("quick brown fox", "field1", "field2") .type(MultiMatchQueryBuilder.Type.CROSS_FIELDS)).endObject()) .get(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", + .setSource(jsonBuilder().startObject() + .field("id", "2") + .field("query", spanNearQuery(spanTermQuery("field1", "quick"), 0) .addClause(spanTermQuery("field1", "brown")) .addClause(spanTermQuery("field1", "fox")) @@ -369,7 +388,9 @@ public void testPercolatorSpecificQueries() throws Exception { client().admin().indices().prepareRefresh().get(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("query", spanNotQuery( spanNearQuery(spanTermQuery("field1", "quick"), 0) .addClause(spanTermQuery("field1", "brown")) @@ -384,7 +405,9 @@ public void testPercolatorSpecificQueries() throws Exception { // doesn't match client().prepareIndex("test").setId("4") - .setSource(jsonBuilder().startObject().field("query", + .setSource(jsonBuilder().startObject() + .field("id", "4") + .field("query", spanNotQuery( spanNearQuery(spanTermQuery("field1", "quick"), 0) .addClause(spanTermQuery("field1", "brown")) @@ -404,7 +427,7 @@ public void testPercolatorSpecificQueries() throws Exception { .endObject()); SearchResponse response = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", source, XContentType.JSON)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 3); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); @@ -424,22 +447,32 @@ public void testPercolatorQueryWithHighlighting() throws Exception { fieldMapping.append(",index_options=offsets"); } assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type", "field1", fieldMapping, "query", "type=percolator") + .addMapping("type", "id", "type=keyword", "field1", fieldMapping, "query", "type=percolator") ); client().prepareIndex("test").setId("1") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "brown fox")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "1") + .field("query", matchQuery("field1", "brown fox")).endObject()) .execute().actionGet(); client().prepareIndex("test").setId("2") - .setSource(jsonBuilder().startObject().field("query", matchQuery("field1", "lazy dog")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "2") + .field("query", matchQuery("field1", "lazy dog")).endObject()) .execute().actionGet(); client().prepareIndex("test").setId("3") - .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "jumps")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "3") + .field("query", termQuery("field1", "jumps")).endObject()) .execute().actionGet(); client().prepareIndex("test").setId("4") - .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "dog")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "4") + .field("query", termQuery("field1", "dog")).endObject()) .execute().actionGet(); client().prepareIndex("test").setId("5") - .setSource(jsonBuilder().startObject().field("query", termQuery("field1", "fox")).endObject()) + .setSource(jsonBuilder().startObject() + .field("id", "5") + .field("query", termQuery("field1", "fox")).endObject()) .execute().actionGet(); client().admin().indices().prepareRefresh().get(); @@ -449,7 +482,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { SearchResponse searchResponse = client().prepareSearch() .setQuery(new PercolateQueryBuilder("query", document, XContentType.JSON)) .highlighter(new HighlightBuilder().field("field1")) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(searchResponse, 5); @@ -476,7 +509,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { .should(new PercolateQueryBuilder("query", document2, XContentType.JSON).setName("query2")) ) .highlighter(new HighlightBuilder().field("field1")) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); logger.info("searchResponse={}", searchResponse); assertHitCount(searchResponse, 5); @@ -500,7 +533,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { BytesReference.bytes(jsonBuilder().startObject().field("field1", "brown fox").endObject()) ), XContentType.JSON)) .highlighter(new HighlightBuilder().field("field1")) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(searchResponse, 5); assertThat(searchResponse.getHits().getAt(0).getFields().get("_percolator_document_slot").getValues(), @@ -540,7 +573,7 @@ public void testPercolatorQueryWithHighlighting() throws Exception { ), XContentType.JSON).setName("query2")) ) .highlighter(new HighlightBuilder().field("field1")) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); logger.info("searchResponse={}", searchResponse); assertHitCount(searchResponse, 5); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index 271cfa630b470..fd9ce3cf8a6f0 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -61,20 +61,20 @@ public void setup() { ensureGreen(); client().prepareIndex(TEST_INDEX).setId("1") - .setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get(); - client().prepareIndex(TEST_INDEX).setId("2").setSource("text", "amsterdam", "population", 851573) + .setSource("id", 1, "text", "berlin", "title", "Berlin, Germany", "population", 3670622).get(); + client().prepareIndex(TEST_INDEX).setId("2").setSource("id", 2, "text", "amsterdam", "population", 851573) .get(); - client().prepareIndex(TEST_INDEX).setId("3").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex(TEST_INDEX).setId("3").setSource("id", 3, "text", "amsterdam", "population", 851573) .get(); - client().prepareIndex(TEST_INDEX).setId("4").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex(TEST_INDEX).setId("4").setSource("id", 4, "text", "amsterdam", "population", 851573) .get(); - client().prepareIndex(TEST_INDEX).setId("5").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex(TEST_INDEX).setId("5").setSource("id", 5, "text", "amsterdam", "population", 851573) .get(); - client().prepareIndex(TEST_INDEX).setId("6").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex(TEST_INDEX).setId("6").setSource("id", 6, "text", "amsterdam", "population", 851573) .get(); // add another index for testing closed indices etc... - client().prepareIndex("test2").setId("7").setSource("text", "amsterdam", "population", 851573) + client().prepareIndex("test2").setId("7").setSource("id", 7, "text", "amsterdam", "population", 851573) .get(); refresh(); @@ -91,7 +91,7 @@ public void testPrecisionAtRequest() { List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - testQuery.sort("_id"); + testQuery.sort("id"); RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", createRelevant("2", "3", "4", "5"), testQuery); amsterdamRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" })); @@ -168,7 +168,7 @@ public void testPrecisionAtRequest() { public void testDCGRequest() { SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - testQuery.sort("_id"); + testQuery.sort("id"); List specifications = new ArrayList<>(); List ratedDocs = Arrays.asList( @@ -202,7 +202,7 @@ public void testDCGRequest() { public void testMRRRequest() { SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - testQuery.sort("_id"); + testQuery.sort("id"); List specifications = new ArrayList<>(); specifications.add(new RatedRequest("amsterdam_query", createRelevant("5"), testQuery)); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyAllocator.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyAllocator.java index 0ec01ebb28b8f..0b9cee7024a00 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyAllocator.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/NettyAllocator.java @@ -44,7 +44,7 @@ public class NettyAllocator { } else { ByteBufAllocator delegate; if (useUnpooled()) { - delegate = new NoDirectBuffers(UnpooledByteBufAllocator.DEFAULT); + delegate = UnpooledByteBufAllocator.DEFAULT; } else { int nHeapArena = PooledByteBufAllocator.defaultNumHeapArena(); int pageSize = PooledByteBufAllocator.defaultPageSize(); diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java index abecbcb1582e8..b87aa8673a197 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperIT.java @@ -65,6 +65,9 @@ public void testBasicUsage() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "tr") @@ -76,8 +79,10 @@ public void testBasicUsage() throws Exception { // both values should collate to same value indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1") + .setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2") + .setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); // searching for either of the terms should return both results since they collate to the same value @@ -87,7 +92,7 @@ public void testBasicUsage() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate") - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); @@ -104,6 +109,9 @@ public void testMultipleValues() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -114,9 +122,10 @@ public void testMultipleValues() throws Exception { // everything should be indexed fine, no exceptions indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":[\"" + equivalent[0] + "\", \"" - + equivalent[1] + "\"]}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[2] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1") + .setSource("{\"id\":\"1\", \"collate\":[\"" + equivalent[0] + "\", \"" + equivalent[1] + "\"]}", XContentType.JSON), + client().prepareIndex(index).setId("2") + .setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[2] + "\"}", XContentType.JSON) ); // using sort mode = max, values B and C will be used for the sort @@ -127,7 +136,7 @@ public void testMultipleValues() throws Exception { .query(QueryBuilders.termQuery("collate", "a")) // if mode max we use c and b as sort values, if max we use "a" for both .sort(SortBuilders.fieldSort("collate").sortMode(SortMode.MAX).order(SortOrder.DESC)) - .sort("_id", SortOrder.DESC) // will be ignored + .sort("id", SortOrder.DESC) // will be ignored ); SearchResponse response = client().search(request).actionGet(); @@ -143,7 +152,7 @@ public void testMultipleValues() throws Exception { .query(QueryBuilders.termQuery("collate", "a")) // if mode max we use c and b as sort values, if max we use "a" for both .sort(SortBuilders.fieldSort("collate").sortMode(SortMode.MIN).order(SortOrder.DESC)) - .sort("_id", SortOrder.DESC) // will NOT be ignored and will determine order + .sort("id", SortOrder.DESC) // will NOT be ignored and will determine order ); response = client().search(request).actionGet(); @@ -163,6 +172,9 @@ public void testNormalization() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "tr") @@ -174,8 +186,10 @@ public void testNormalization() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1") + .setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2") + .setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); // searching for either of the terms should return both results since they collate to the same value @@ -185,7 +199,7 @@ public void testNormalization() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate") - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); @@ -205,6 +219,9 @@ public void testSecondaryStrength() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -216,8 +233,10 @@ public void testSecondaryStrength() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1") + .setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2") + .setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -226,7 +245,7 @@ public void testSecondaryStrength() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate") - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); @@ -247,6 +266,9 @@ public void testIgnorePunctuation() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -258,8 +280,8 @@ public void testIgnorePunctuation() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -268,7 +290,7 @@ public void testIgnorePunctuation() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate") - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); @@ -287,6 +309,9 @@ public void testIgnoreWhitespace() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -300,9 +325,9 @@ public void testIgnoreWhitespace() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"foo bar\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"foobar\"}", XContentType.JSON), - client().prepareIndex(index).setId("3").setSource("{\"collate\":\"foo-bar\"}", XContentType.JSON) + client().prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"foo bar\"}", XContentType.JSON), + client().prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"foobar\"}", XContentType.JSON), + client().prepareIndex(index).setId("3").setSource("{\"id\":\"3\",\"collate\":\"foo-bar\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -310,7 +335,7 @@ public void testIgnoreWhitespace() throws Exception { .source(new SearchSourceBuilder() .fetchSource(false) .sort("collate", SortOrder.ASC) - .sort("_id", SortOrder.ASC) // secondary sort should kick in on docs 1 and 3 because same value collate value + .sort("id", SortOrder.ASC) // secondary sort should kick in on docs 1 and 3 because same value collate value ); SearchResponse response = client().search(request).actionGet(); @@ -367,6 +392,9 @@ public void testIgnoreAccentsButNotCase() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("language", "en") @@ -379,10 +407,10 @@ public void testIgnoreAccentsButNotCase() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"résumé\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"Resume\"}", XContentType.JSON), - client().prepareIndex(index).setId("3").setSource("{\"collate\":\"resume\"}", XContentType.JSON), - client().prepareIndex(index).setId("4").setSource("{\"collate\":\"Résumé\"}", XContentType.JSON) + client().prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"résumé\"}", XContentType.JSON), + client().prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"Resume\"}", XContentType.JSON), + client().prepareIndex(index).setId("3").setSource("{\"id\":\"3\",\"collate\":\"resume\"}", XContentType.JSON), + client().prepareIndex(index).setId("4").setSource("{\"id\":\"4\",\"collate\":\"Résumé\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -390,7 +418,7 @@ public void testIgnoreAccentsButNotCase() throws Exception { .source(new SearchSourceBuilder() .fetchSource(false) .sort("collate", SortOrder.ASC) - .sort("_id", SortOrder.DESC) + .sort("id", SortOrder.DESC) ); SearchResponse response = client().search(request).actionGet(); @@ -462,6 +490,9 @@ public void testCustomRules() throws Exception { XContentBuilder builder = jsonBuilder() .startObject().startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("collate") .field("type", "icu_collation_keyword") .field("rules", tailoredRules) @@ -472,8 +503,8 @@ public void testCustomRules() throws Exception { assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); indexRandom(true, - client().prepareIndex(index).setId("1").setSource("{\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), - client().prepareIndex(index).setId("2").setSource("{\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) + client().prepareIndex(index).setId("1").setSource("{\"id\":\"1\",\"collate\":\"" + equivalent[0] + "\"}", XContentType.JSON), + client().prepareIndex(index).setId("2").setSource("{\"id\":\"2\",\"collate\":\"" + equivalent[1] + "\"}", XContentType.JSON) ); SearchRequest request = new SearchRequest() @@ -482,7 +513,7 @@ public void testCustomRules() throws Exception { .fetchSource(false) .query(QueryBuilders.termQuery("collate", randomBoolean() ? equivalent[0] : equivalent[1])) .sort("collate", SortOrder.ASC) - .sort("_id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value + .sort("id", SortOrder.DESC) // secondary sort should kick in because both will collate to same value ); SearchResponse response = client().search(request).actionGet(); diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index 7b195bdc7b434..60036511e2f43 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -31,13 +31,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.NormsFieldExistsQuery; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper; -import org.apache.lucene.search.spans.SpanQuery; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -45,15 +38,12 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; -import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText.AnnotationToken; -import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; import java.io.IOException; @@ -531,7 +521,7 @@ private void emitAnnotation(int firstSpannedTextPosInc, int annotationPosLen) th } - public static final class AnnotatedTextFieldType extends StringFieldType { + public static final class AnnotatedTextFieldType extends TextFieldMapper.TextFieldType { public AnnotatedTextFieldType() { setTokenized(true); @@ -562,37 +552,6 @@ public String typeName() { return CONTENT_TYPE; } - @Override - public Query existsQuery(QueryShardContext context) { - if (omitNorms()) { - return new TermQuery(new Term(FieldNamesFieldMapper.NAME, name())); - } else { - return new NormsFieldExistsQuery(name()); - } - } - - @Override - public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRewriteMethod method, QueryShardContext context) { - SpanMultiTermQueryWrapper spanMulti = - new SpanMultiTermQueryWrapper<>(new PrefixQuery(new Term(name(), indexedValueForSearch(value)))); - spanMulti.setRewriteMethod(method); - return spanMulti; - } - - @Override - public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { - return TextFieldMapper.createPhraseQuery(stream, name(), slop, enablePositionIncrements); - } - - @Override - public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements) throws IOException { - return TextFieldMapper.createPhraseQuery(stream, name(), slop, enablePositionIncrements); - } - - @Override - public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions) throws IOException { - return TextFieldMapper.createPhrasePrefixQuery(stream, name(), slop, maxExpansions, null, null); - } } private int positionIncrementGap; diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 5e50bd3898674..5acc8c9a82280 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -672,6 +672,4 @@ public void testEmptyName() throws IOException { assertThat(e.getMessage(), containsString("name cannot be empty string")); } - - } diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java new file mode 100644 index 0000000000000..0bbc25e6171b8 --- /dev/null +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.annotatedtext; + +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.queries.intervals.Intervals; +import org.apache.lucene.queries.intervals.IntervalsSource; +import org.elasticsearch.index.analysis.AnalyzerScope; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; + +import java.io.IOException; + +public class AnnotatedTextFieldTypeTests extends FieldTypeTestCase { + @Override + protected MappedFieldType createDefaultFieldType() { + return new AnnotatedTextFieldMapper.AnnotatedTextFieldType(); + } + + public void testIntervals() throws IOException { + MappedFieldType ft = createDefaultFieldType(); + NamedAnalyzer a = new NamedAnalyzer("name", AnalyzerScope.INDEX, new StandardAnalyzer()); + IntervalsSource source = ft.intervals("Donald Trump", 0, true, a, false); + assertEquals(Intervals.phrase(Intervals.term("donald"), Intervals.term("trump")), source); + } +} diff --git a/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml b/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml index aca6ba3059381..b4acccf36879d 100644 --- a/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml +++ b/plugins/mapper-annotated-text/src/test/resources/rest-api-spec/test/mapper_annotatedtext/10_basic.yml @@ -39,6 +39,41 @@ - match: {hits.hits.0.highlight.text.0: "The [quick](_hit_term=quick) brown fox is brown."} + - do: + search: + body: + query: + intervals: + text: + match: + query: entity_3789 brown + + - match: { hits.total.value: 1 } + + - do: + search: + body: + query: + span_near: + clauses: [ + span_term: { text: entity_3789 }, + span_term: { text: brown } + ] + in_order: true + slop: 10 + + - match: { hits.total.value: 1 } + + - do: + search: + body: + query: + match_phrase: + text: "fox is brown" + + - match: { hits.total.value: 1 } + + --- "issue 39395 thread safety issue -requires multiple calls to reveal": - do: @@ -57,13 +92,13 @@ index: index: annotated id: 1 - body: + body: "my_field" : "[A](~MARK0&~MARK0) [B](~MARK1)" - do: index: index: annotated id: 2 - body: + body: "my_field" : "[A](~MARK0) [C](~MARK2)" refresh: true - do: diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java index 463437597a7fd..1c069ca189951 100644 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java +++ b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobContainerRetriesTests.java @@ -72,7 +72,7 @@ import java.util.stream.Collectors; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; +import static org.elasticsearch.repositories.ESBlobStoreContainerTestCase.randomBytes; import static org.elasticsearch.repositories.azure.AzureRepository.Repository.CONTAINER_SETTING; import static org.elasticsearch.repositories.azure.AzureStorageSettings.ACCOUNT_SETTING; import static org.elasticsearch.repositories.azure.AzureStorageSettings.ENDPOINT_SUFFIX_SETTING; diff --git a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java b/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java deleted file mode 100644 index 74bfcb784aed0..0000000000000 --- a/plugins/repository-azure/src/test/java/org/elasticsearch/repositories/azure/AzureBlobStoreTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.repositories.azure; - -import org.elasticsearch.cluster.metadata.RepositoryMetaData; -import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.repositories.ESBlobStoreTestCase; -import org.elasticsearch.threadpool.TestThreadPool; -import org.elasticsearch.threadpool.ThreadPool; - -import java.util.concurrent.TimeUnit; - -public class AzureBlobStoreTests extends ESBlobStoreTestCase { - - private ThreadPool threadPool; - - @Override - public void setUp() throws Exception { - super.setUp(); - threadPool = new TestThreadPool("AzureBlobStoreTests", AzureRepositoryPlugin.executorBuilder()); - } - - @Override - public void tearDown() throws Exception { - super.tearDown(); - ThreadPool.terminate(threadPool, 10L, TimeUnit.SECONDS); - } - - @Override - protected BlobStore newBlobStore() { - RepositoryMetaData repositoryMetaData = new RepositoryMetaData("azure", "ittest", Settings.EMPTY); - AzureStorageServiceMock client = new AzureStorageServiceMock(); - return new AzureBlobStore(repositoryMetaData, client, threadPool); - } -} diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java index f70625dd179b5..00dbf758422f7 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobContainerRetriesTests.java @@ -74,7 +74,7 @@ import static fixture.gcs.GoogleCloudStorageHttpHandler.getContentRangeStart; import static fixture.gcs.GoogleCloudStorageHttpHandler.parseMultipartRequestBody; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; +import static org.elasticsearch.repositories.ESBlobStoreContainerTestCase.randomBytes; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING; import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING; diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java index cc3782cabac2c..311544160ad73 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreContainerTests.java @@ -41,7 +41,6 @@ import java.util.Locale; import java.util.concurrent.ConcurrentHashMap; -import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java index 7c1194627ac33..ee0b59eb9d365 100644 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java +++ b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreRepositoryTests.java @@ -212,7 +212,7 @@ protected String requestUniqueId(HttpExchange exchange) { if ("/token".equals(exchange.getRequestURI().getPath())) { try { // token content is unique per node (not per request) - return Streams.readFully(exchange.getRequestBody()).utf8ToString(); + return Streams.readFully(Streams.noCloseStream(exchange.getRequestBody())).utf8ToString(); } catch (IOException e) { throw new AssertionError("Unable to read token request body", e); } diff --git a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java b/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java deleted file mode 100644 index 294adfdaec573..0000000000000 --- a/plugins/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageBlobStoreTests.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.gcs; - -import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.repositories.ESBlobStoreTestCase; - -import java.util.Locale; -import java.util.concurrent.ConcurrentHashMap; - -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class GoogleCloudStorageBlobStoreTests extends ESBlobStoreTestCase { - - @Override - protected BlobStore newBlobStore() { - final String bucketName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); - final String clientName = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); - final GoogleCloudStorageService storageService = mock(GoogleCloudStorageService.class); - try { - when(storageService.client(any(String.class))).thenReturn(new MockStorage(bucketName, new ConcurrentHashMap<>(), random())); - } catch (final Exception e) { - throw new RuntimeException(e); - } - return new GoogleCloudStorageBlobStore(bucketName, clientName, storageService); - } -} diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java index c0b0b7a307d49..83e5c581e065c 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java @@ -45,10 +45,6 @@ import java.security.PrivilegedExceptionAction; import java.util.Collections; -import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; -import static org.elasticsearch.repositories.ESBlobStoreTestCase.readBlobFully; - - @ThreadLeakFilters(filters = {HdfsClientThreadLeakFilter.class}) public class HdfsBlobStoreContainerTests extends ESBlobStoreContainerTestCase { diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java index 422fab45e43bc..367ed06dc6551 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreContainerTests.java @@ -34,10 +34,14 @@ import com.amazonaws.services.s3.model.StorageClass; import com.amazonaws.services.s3.model.UploadPartRequest; import com.amazonaws.services.s3.model.UploadPartResult; +import org.elasticsearch.cluster.metadata.RepositoryMetaData; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.blobstore.BlobStoreException; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; +import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.repositories.ESBlobStoreContainerTestCase; import org.mockito.ArgumentCaptor; @@ -46,10 +50,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; +import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import java.util.stream.IntStream; -import static org.elasticsearch.repositories.s3.S3BlobStoreTests.randomMockS3BlobStore; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.mockito.Matchers.any; import static org.mockito.Mockito.doNothing; @@ -397,10 +403,99 @@ public void testNumberOfMultiparts() { assertNumberOfMultiparts(factor + 1, remaining, (size * factor) + remaining, size); } + public void testInitCannedACL() { + String[] aclList = new String[]{ + "private", "public-read", "public-read-write", "authenticated-read", + "log-delivery-write", "bucket-owner-read", "bucket-owner-full-control"}; + + //empty acl + assertThat(S3BlobStore.initCannedACL(null), equalTo(CannedAccessControlList.Private)); + assertThat(S3BlobStore.initCannedACL(""), equalTo(CannedAccessControlList.Private)); + + // it should init cannedACL correctly + for (String aclString : aclList) { + CannedAccessControlList acl = S3BlobStore.initCannedACL(aclString); + assertThat(acl.toString(), equalTo(aclString)); + } + + // it should accept all aws cannedACLs + for (CannedAccessControlList awsList : CannedAccessControlList.values()) { + CannedAccessControlList acl = S3BlobStore.initCannedACL(awsList.toString()); + assertThat(acl, equalTo(awsList)); + } + } + + public void testInvalidCannedACL() { + BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initCannedACL("test_invalid")); + assertThat(ex.getMessage(), equalTo("cannedACL is not valid: [test_invalid]")); + } + + public void testInitStorageClass() { + // it should default to `standard` + assertThat(S3BlobStore.initStorageClass(null), equalTo(StorageClass.Standard)); + assertThat(S3BlobStore.initStorageClass(""), equalTo(StorageClass.Standard)); + + // it should accept [standard, standard_ia, onezone_ia, reduced_redundancy, intelligent_tiering] + assertThat(S3BlobStore.initStorageClass("standard"), equalTo(StorageClass.Standard)); + assertThat(S3BlobStore.initStorageClass("standard_ia"), equalTo(StorageClass.StandardInfrequentAccess)); + assertThat(S3BlobStore.initStorageClass("onezone_ia"), equalTo(StorageClass.OneZoneInfrequentAccess)); + assertThat(S3BlobStore.initStorageClass("reduced_redundancy"), equalTo(StorageClass.ReducedRedundancy)); + assertThat(S3BlobStore.initStorageClass("intelligent_tiering"), equalTo(StorageClass.IntelligentTiering)); + } + + public void testCaseInsensitiveStorageClass() { + assertThat(S3BlobStore.initStorageClass("sTandaRd"), equalTo(StorageClass.Standard)); + assertThat(S3BlobStore.initStorageClass("sTandaRd_Ia"), equalTo(StorageClass.StandardInfrequentAccess)); + assertThat(S3BlobStore.initStorageClass("oNeZoNe_iA"), equalTo(StorageClass.OneZoneInfrequentAccess)); + assertThat(S3BlobStore.initStorageClass("reduCED_redundancy"), equalTo(StorageClass.ReducedRedundancy)); + assertThat(S3BlobStore.initStorageClass("intelLigeNt_tieriNG"), equalTo(StorageClass.IntelligentTiering)); + } + + public void testInvalidStorageClass() { + BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initStorageClass("whatever")); + assertThat(ex.getMessage(), equalTo("`whatever` is not a valid S3 Storage Class.")); + } + + public void testRejectGlacierStorageClass() { + BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initStorageClass("glacier")); + assertThat(ex.getMessage(), equalTo("Glacier storage class is not supported")); + } + private static void assertNumberOfMultiparts(final int expectedParts, final long expectedRemaining, long totalSize, long partSize) { final Tuple result = S3BlobContainer.numberOfMultiparts(totalSize, partSize); assertEquals("Expected number of parts [" + expectedParts + "] but got [" + result.v1() + "]", expectedParts, (long) result.v1()); assertEquals("Expected remaining [" + expectedRemaining + "] but got [" + result.v2() + "]", expectedRemaining, (long) result.v2()); } + + /** + * Creates a new {@link S3BlobStore} with random settings. + *

+ * The blobstore uses a {@link MockAmazonS3} client. + */ + public static S3BlobStore randomMockS3BlobStore() { + String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); + ByteSizeValue bufferSize = new ByteSizeValue(randomIntBetween(5, 100), ByteSizeUnit.MB); + boolean serverSideEncryption = randomBoolean(); + + String cannedACL = null; + if (randomBoolean()) { + cannedACL = randomFrom(CannedAccessControlList.values()).toString(); + } + + String storageClass = null; + if (randomBoolean()) { + storageClass = randomValueOtherThan(StorageClass.Glacier, () -> randomFrom(StorageClass.values())).toString(); + } + + final AmazonS3 client = new MockAmazonS3(new ConcurrentHashMap<>(), bucket, serverSideEncryption, cannedACL, storageClass); + final S3Service service = new S3Service() { + @Override + public synchronized AmazonS3Reference client(RepositoryMetaData repositoryMetaData) { + return new AmazonS3Reference(client); + } + }; + return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, + new RepositoryMetaData(bucket, "s3", Settings.EMPTY)); + } } diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java deleted file mode 100644 index 076ef4864a08d..0000000000000 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/S3BlobStoreTests.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.repositories.s3; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.CannedAccessControlList; -import com.amazonaws.services.s3.model.StorageClass; -import org.elasticsearch.cluster.metadata.RepositoryMetaData; -import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.blobstore.BlobStoreException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.repositories.ESBlobStoreTestCase; - -import java.util.Locale; -import java.util.concurrent.ConcurrentHashMap; - -import static org.hamcrest.Matchers.equalTo; - -public class S3BlobStoreTests extends ESBlobStoreTestCase { - - @Override - protected BlobStore newBlobStore() { - return randomMockS3BlobStore(); - } - - public void testInitCannedACL() { - String[] aclList = new String[]{ - "private", "public-read", "public-read-write", "authenticated-read", - "log-delivery-write", "bucket-owner-read", "bucket-owner-full-control"}; - - //empty acl - assertThat(S3BlobStore.initCannedACL(null), equalTo(CannedAccessControlList.Private)); - assertThat(S3BlobStore.initCannedACL(""), equalTo(CannedAccessControlList.Private)); - - // it should init cannedACL correctly - for (String aclString : aclList) { - CannedAccessControlList acl = S3BlobStore.initCannedACL(aclString); - assertThat(acl.toString(), equalTo(aclString)); - } - - // it should accept all aws cannedACLs - for (CannedAccessControlList awsList : CannedAccessControlList.values()) { - CannedAccessControlList acl = S3BlobStore.initCannedACL(awsList.toString()); - assertThat(acl, equalTo(awsList)); - } - } - - public void testInvalidCannedACL() { - BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initCannedACL("test_invalid")); - assertThat(ex.getMessage(), equalTo("cannedACL is not valid: [test_invalid]")); - } - - public void testInitStorageClass() { - // it should default to `standard` - assertThat(S3BlobStore.initStorageClass(null), equalTo(StorageClass.Standard)); - assertThat(S3BlobStore.initStorageClass(""), equalTo(StorageClass.Standard)); - - // it should accept [standard, standard_ia, onezone_ia, reduced_redundancy, intelligent_tiering] - assertThat(S3BlobStore.initStorageClass("standard"), equalTo(StorageClass.Standard)); - assertThat(S3BlobStore.initStorageClass("standard_ia"), equalTo(StorageClass.StandardInfrequentAccess)); - assertThat(S3BlobStore.initStorageClass("onezone_ia"), equalTo(StorageClass.OneZoneInfrequentAccess)); - assertThat(S3BlobStore.initStorageClass("reduced_redundancy"), equalTo(StorageClass.ReducedRedundancy)); - assertThat(S3BlobStore.initStorageClass("intelligent_tiering"), equalTo(StorageClass.IntelligentTiering)); - } - - public void testCaseInsensitiveStorageClass() { - assertThat(S3BlobStore.initStorageClass("sTandaRd"), equalTo(StorageClass.Standard)); - assertThat(S3BlobStore.initStorageClass("sTandaRd_Ia"), equalTo(StorageClass.StandardInfrequentAccess)); - assertThat(S3BlobStore.initStorageClass("oNeZoNe_iA"), equalTo(StorageClass.OneZoneInfrequentAccess)); - assertThat(S3BlobStore.initStorageClass("reduCED_redundancy"), equalTo(StorageClass.ReducedRedundancy)); - assertThat(S3BlobStore.initStorageClass("intelLigeNt_tieriNG"), equalTo(StorageClass.IntelligentTiering)); - } - - public void testInvalidStorageClass() { - BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initStorageClass("whatever")); - assertThat(ex.getMessage(), equalTo("`whatever` is not a valid S3 Storage Class.")); - } - - public void testRejectGlacierStorageClass() { - BlobStoreException ex = expectThrows(BlobStoreException.class, () -> S3BlobStore.initStorageClass("glacier")); - assertThat(ex.getMessage(), equalTo("Glacier storage class is not supported")); - } - - /** - * Creates a new {@link S3BlobStore} with random settings. - *

- * The blobstore uses a {@link MockAmazonS3} client. - */ - public static S3BlobStore randomMockS3BlobStore() { - String bucket = randomAlphaOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT); - ByteSizeValue bufferSize = new ByteSizeValue(randomIntBetween(5, 100), ByteSizeUnit.MB); - boolean serverSideEncryption = randomBoolean(); - - String cannedACL = null; - if (randomBoolean()) { - cannedACL = randomFrom(CannedAccessControlList.values()).toString(); - } - - String storageClass = null; - if (randomBoolean()) { - storageClass = randomValueOtherThan(StorageClass.Glacier, () -> randomFrom(StorageClass.values())).toString(); - } - - final AmazonS3 client = new MockAmazonS3(new ConcurrentHashMap<>(), bucket, serverSideEncryption, cannedACL, storageClass); - final S3Service service = new S3Service() { - @Override - public synchronized AmazonS3Reference client(RepositoryMetaData repositoryMetaData) { - return new AmazonS3Reference(client); - } - }; - return new S3BlobStore(service, bucket, serverSideEncryption, bufferSize, cannedACL, storageClass, - new RepositoryMetaData(bucket, "s3", Settings.EMPTY)); - } -} diff --git a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java index 558e6071255b1..a4e76fc7ec436 100644 --- a/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java +++ b/qa/multi-cluster-search/src/test/java/org/elasticsearch/search/CCSDuelIT.java @@ -165,7 +165,7 @@ private static void indexDocuments(String idPrefix) throws IOException, Interrup //this index with a single document is used to test partial failures IndexRequest indexRequest = new IndexRequest(INDEX_NAME + "_err"); indexRequest.id("id"); - indexRequest.source("creationDate", "err"); + indexRequest.source("id", "id", "creationDate", "err"); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL); IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT); assertEquals(201, indexResponse.status().getStatus()); @@ -178,6 +178,7 @@ private static void indexDocuments(String idPrefix) throws IOException, Interrup CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME); createIndexRequest.settings(Settings.builder().put("index.number_of_shards", numShards).put("index.number_of_replicas", 0)); createIndexRequest.mapping("{\"properties\":{" + + "\"id\":{\"type\":\"keyword\"}," + "\"suggest\":{\"type\":\"completion\"}," + "\"join\":{\"type\":\"join\", \"relations\": {\"question\":\"answer\"}}}}", XContentType.JSON); CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(createIndexRequest, RequestOptions.DEFAULT); @@ -237,6 +238,7 @@ private static IndexRequest buildIndexRequest(String id, String type, String que joinField.put("parent", questionId); } indexRequest.source(XContentType.JSON, + "id", id, "type", type, "votes", randomIntBetween(0, 30), "questionId", questionId, @@ -614,7 +616,7 @@ public void testTopHits() throws Exception { topHits.from(10); topHits.size(10); topHits.sort("creationDate", SortOrder.DESC); - topHits.sort("_id", SortOrder.ASC); + topHits.sort("id", SortOrder.ASC); TermsAggregationBuilder tags = new TermsAggregationBuilder("tags", ValueType.STRING); tags.field("tags.keyword"); tags.size(10); diff --git a/qa/os/bats/default/20_tar_bootstrap_password.bats b/qa/os/bats/default/20_tar_bootstrap_password.bats deleted file mode 120000 index 58a968aa3e14c..0000000000000 --- a/qa/os/bats/default/20_tar_bootstrap_password.bats +++ /dev/null @@ -1 +0,0 @@ -bootstrap_password.bash \ No newline at end of file diff --git a/qa/os/bats/default/25_package_bootstrap_password.bats b/qa/os/bats/default/25_package_bootstrap_password.bats deleted file mode 120000 index 58a968aa3e14c..0000000000000 --- a/qa/os/bats/default/25_package_bootstrap_password.bats +++ /dev/null @@ -1 +0,0 @@ -bootstrap_password.bash \ No newline at end of file diff --git a/qa/os/bats/default/30_tar_setup_passwords.bats b/qa/os/bats/default/30_tar_setup_passwords.bats deleted file mode 120000 index 74d1204b3f9e7..0000000000000 --- a/qa/os/bats/default/30_tar_setup_passwords.bats +++ /dev/null @@ -1 +0,0 @@ -setup_passwords.bash \ No newline at end of file diff --git a/qa/os/bats/default/35_package_setup_passwords.bats b/qa/os/bats/default/35_package_setup_passwords.bats deleted file mode 120000 index 74d1204b3f9e7..0000000000000 --- a/qa/os/bats/default/35_package_setup_passwords.bats +++ /dev/null @@ -1 +0,0 @@ -setup_passwords.bash \ No newline at end of file diff --git a/qa/os/bats/default/bootstrap_password.bash b/qa/os/bats/default/bootstrap_password.bash deleted file mode 100644 index 61d4e4315da5d..0000000000000 --- a/qa/os/bats/default/bootstrap_password.bash +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env bats - -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License; -# you may not use this file except in compliance with the Elastic License. - -load $BATS_UTILS/utils.bash -load $BATS_UTILS/plugins.bash -load $BATS_UTILS/xpack.bash - -setup() { - if [ $BATS_TEST_NUMBER == 1 ]; then - export PACKAGE_NAME="elasticsearch" - clean_before_test - install - set_debug_logging - - generate_trial_license - verify_xpack_installation - fi -} - -if [[ "$BATS_TEST_FILENAME" =~ 20_tar_bootstrap_password.bats$ ]]; then - load $BATS_UTILS/tar.bash - GROUP='TAR BOOTSTRAP PASSWORD' - install() { - install_archive - verify_archive_installation - } - export ESHOME=/tmp/elasticsearch - export_elasticsearch_paths - export ESPLUGIN_COMMAND_USER=elasticsearch -else - load $BATS_UTILS/packages.bash - if is_rpm; then - GROUP='RPM BOOTSTRAP PASSWORD' - elif is_dpkg; then - GROUP='DEB BOOTSTRAP PASSWORD' - fi - export_elasticsearch_paths - export ESPLUGIN_COMMAND_USER=root - install() { - install_package - verify_package_installation - } -fi - -@test "[$GROUP] add bootstrap.password setting" { - if [[ -f /tmp/bootstrap.password ]]; then - sudo rm -f /tmp/bootstrap.password - fi - - run sudo -E -u $ESPLUGIN_COMMAND_USER bash <<"NEW_PASS" -if [[ ! -f $ESCONFIG/elasticsearch.keystore ]]; then - $ESHOME/bin/elasticsearch-keystore create -fi -cat /dev/urandom | tr -dc "[a-zA-Z0-9]" | fold -w 20 | head -n 1 > /tmp/bootstrap.password -cat /tmp/bootstrap.password | $ESHOME/bin/elasticsearch-keystore add --stdin bootstrap.password -NEW_PASS - [ "$status" -eq 0 ] || { - echo "Expected elasticsearch-keystore tool exit code to be zero but got [$status]" - echo "$output" - false - } - assert_file_exist "/tmp/bootstrap.password" -} - -@test "[$GROUP] test bootstrap.password is in setting list" { - run sudo -E -u $ESPLUGIN_COMMAND_USER bash <<"NODE_SETTINGS" -cat >> $ESCONFIG/elasticsearch.yml <<- EOF -network.host: 127.0.0.1 -http.port: 9200 -EOF -NODE_SETTINGS - - run_elasticsearch_service 0 - wait_for_xpack 127.0.0.1 9200 - - sudo -E -u $ESPLUGIN_COMMAND_USER "$ESHOME/bin/elasticsearch-keystore" list | grep "bootstrap.password" - - password=$(cat /tmp/bootstrap.password) - clusterHealth=$(sudo curl -u "elastic:$password" -H "Content-Type: application/json" \ - -XGET "http://127.0.0.1:9200/_cluster/health?wait_for_status=green&timeout=180s") - echo "$clusterHealth" | grep '"status":"green"' || { - echo "Expected cluster health to be green but got:" - echo "$clusterHealth" - false - } -} - -@test "[$GROUP] test auto generated passwords with modified bootstrap.password" { - if [[ -f /tmp/setup-passwords-output-with-bootstrap ]]; then - sudo rm -f /tmp/setup-passwords-output-with-bootstrap - fi - - run sudo -E -u $ESPLUGIN_COMMAND_USER bash <<"SETUP_OK" -echo 'y' | $ESHOME/bin/elasticsearch-setup-passwords auto -SETUP_OK - echo "$output" > /tmp/setup-passwords-output-with-bootstrap - [ "$status" -eq 0 ] || { - echo "Expected x-pack elasticsearch-setup-passwords tool exit code to be zero but got [$status]" - cat /tmp/setup-passwords-output-with-bootstrap - debug_collect_logs - false - } - - curl -s -XGET 'http://127.0.0.1:9200' | grep "missing authentication credentials for REST" - - # Disable bash history expansion because passwords can contain "!" - set +H - - users=( elastic kibana logstash_system ) - for user in "${users[@]}"; do - grep "Changed password for user $user" /tmp/setup-passwords-output-with-bootstrap || { - echo "Expected x-pack elasticsearch-setup-passwords tool to change password for user [$user]:" - cat /tmp/setup-passwords-output-with-bootstrap - false - } - - password=$(grep "PASSWORD $user = " /tmp/setup-passwords-output-with-bootstrap | sed "s/PASSWORD $user = //") - curl -u "$user:$password" -XGET 'http://127.0.0.1:9200' | grep "You Know, for Search" - - basic=$(echo -n "$user:$password" | base64) - curl -H "Authorization: Basic $basic" -XGET 'http://127.0.0.1:9200' | grep "You Know, for Search" - done - set -H -} - -@test "[$GROUP] test elasticsearch-sql-cli" { - password=$(grep "PASSWORD elastic = " /tmp/setup-passwords-output-with-bootstrap | sed "s/PASSWORD elastic = //") - curl -s -u "elastic:$password" -H "Content-Type: application/json" -XPUT 'localhost:9200/library/_doc/1?refresh&pretty' -d'{ - "name": "Ender'"'"'s Game", - "author": "Orson Scott Card", - "release_date": "1985-06-01", - "page_count": 324 - }' - - password=$(grep "PASSWORD elastic = " /tmp/setup-passwords-output-with-bootstrap | sed "s/PASSWORD elastic = //") - - run $ESHOME/bin/elasticsearch-sql-cli --debug "http://elastic@127.0.0.1:9200" < /tmp/setup-passwords-output - [ "$status" -eq 0 ] || { - echo "Expected x-pack elasticsearch-setup-passwords tool exit code to be zero but got $status" - cat /tmp/setup-passwords-output - debug_collect_logs - false - } - - curl -s -XGET localhost:9200 | grep "missing authentication credentials for REST" - - # Disable bash history expansion because passwords can contain "!" - set +H - - users=( elastic kibana logstash_system ) - for user in "${users[@]}"; do - grep "Changed password for user $user" /tmp/setup-passwords-output || { - echo "Expected x-pack elasticsearch-setup-passwords tool to change password for user [$user]:" - cat /tmp/setup-passwords-output - false - } - - password=$(grep "PASSWORD $user = " /tmp/setup-passwords-output | sed "s/PASSWORD $user = //") - curl -u "$user:$password" -XGET localhost:9200 | grep "You Know, for Search" - - basic=$(echo -n "$user:$password" | base64) - curl -H "Authorization: Basic $basic" -XGET localhost:9200 | grep "You Know, for Search" - done - set -H - - stop_elasticsearch_service - assert_file_not_exist "/home/elasticsearch" -} - -@test "[$GROUP] remove Elasticsearch" { - # NOTE: this must be the last test, so that running oss tests does not already have the default distro still installed - clean_before_test -} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 4b47eb8511eb3..37fd4448974af 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -69,7 +69,7 @@ public void test10Install() throws Exception { public void test20PluginsListWithNoPlugins() throws Exception { final Installation.Executables bin = installation.executables(); - final Result r = sh.run(bin.elasticsearchPlugin + " list"); + final Result r = bin.elasticsearchPlugin.run(sh, "list"); assertThat(r.stdout, isEmptyString()); } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 997d4b9758684..e75bfc8ad38df 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -40,7 +40,9 @@ import static org.elasticsearch.packaging.util.Docker.copyFromContainer; import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded; import static org.elasticsearch.packaging.util.Docker.existsInContainer; +import static org.elasticsearch.packaging.util.Docker.mkDirWithPrivilegeEscalation; import static org.elasticsearch.packaging.util.Docker.removeContainer; +import static org.elasticsearch.packaging.util.Docker.rmDirWithPrivilegeEscalation; import static org.elasticsearch.packaging.util.Docker.runContainer; import static org.elasticsearch.packaging.util.Docker.runContainerExpectingFailure; import static org.elasticsearch.packaging.util.Docker.verifyContainerInstallation; @@ -177,6 +179,27 @@ public void test70BindMountCustomPathConfAndJvmOptions() throws Exception { assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); } + /** + * Check that the default config can be overridden using a bind mount, and that env vars are respected + */ + public void test71BindMountCustomPathWithDifferentUID() throws Exception { + final Path tempEsDataDir = tempDir.resolve("esDataDir"); + // Make the local directory and contents accessible when bind-mounted + mkDirWithPrivilegeEscalation(tempEsDataDir, 1500, 0); + + // Restart the container + final Map volumes = Map.of(tempEsDataDir.toAbsolutePath(), installation.data); + + runContainer(distribution(), volumes, null); + + waitForElasticsearch(installation); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + + assertThat(nodesResponse, containsString("\"_nodes\":{\"total\":1,\"successful\":1,\"failed\":0}")); + rmDirWithPrivilegeEscalation(tempEsDataDir); + } + /** * Check that environment variables can be populated by setting variables with the suffix "_FILE", * which point to files that hold the required values. diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 3efd1b36ddbdd..cb95408b2a5bf 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -26,8 +26,12 @@ import com.carrotsearch.randomizedtesting.annotations.Timeout; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.packaging.util.Archives; import org.elasticsearch.packaging.util.Distribution; +import org.elasticsearch.packaging.util.Docker; +import org.elasticsearch.packaging.util.FileUtils; import org.elasticsearch.packaging.util.Installation; +import org.elasticsearch.packaging.util.Packages; import org.elasticsearch.packaging.util.Platforms; import org.elasticsearch.packaging.util.Shell; import org.junit.Assert; @@ -40,6 +44,7 @@ import org.junit.runner.Description; import org.junit.runner.RunWith; +import java.nio.file.Files; import java.nio.file.Paths; import static org.elasticsearch.packaging.util.Cleanup.cleanEverything; @@ -119,7 +124,78 @@ protected static Distribution distribution() { return distribution; } - protected Shell newShell() throws Exception { + protected static void install() throws Exception { + switch (distribution.packaging) { + case TAR: + case ZIP: + installation = Archives.installArchive(distribution); + Archives.verifyArchiveInstallation(installation, distribution); + break; + case DEB: + case RPM: + installation = Packages.installPackage(distribution); + Packages.verifyPackageInstallation(installation, distribution, newShell()); + break; + case DOCKER: + installation = Docker.runContainer(distribution); + Docker.verifyContainerInstallation(installation, distribution); + } + } + + /** + * Starts and stops elasticsearch, and performs assertions while it is running. + */ + protected void assertWhileRunning(Platforms.PlatformAction assertions) throws Exception { + try { + switch (distribution.packaging) { + case TAR: + case ZIP: + Archives.runElasticsearch(installation, sh); + break; + case DEB: + case RPM: + Packages.startElasticsearch(sh, installation); + break; + case DOCKER: + // nothing, "installing" docker image is running it + } + + } catch (Exception e ){ + if (Files.exists(installation.home.resolve("elasticsearch.pid"))) { + String pid = FileUtils.slurp(installation.home.resolve("elasticsearch.pid")).trim(); + logger.info("Dumping jstack of elasticsearch processb ({}) that failed to start", pid); + sh.runIgnoreExitCode("jstack " + pid); + } + if (Files.exists(installation.logs.resolve("elasticsearch.log"))) { + logger.warn("Elasticsearch log:\n" + + FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz")); + } + throw e; + } + + try { + assertions.run(); + } catch (Exception e) { + logger.warn("Elasticsearch log:\n" + + FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz")); + throw e; + } + + switch (distribution.packaging) { + case TAR: + case ZIP: + Archives.stopElasticsearch(installation); + break; + case DEB: + case RPM: + Packages.stopElasticsearch(sh); + break; + case DOCKER: + // nothing, removing container is handled externally + } + } + + protected static Shell newShell() throws Exception { Shell sh = new Shell(); if (distribution().hasJdk == false) { Platforms.onLinux(() -> { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java new file mode 100644 index 0000000000000..9082b19f0bd49 --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PasswordToolsTests.java @@ -0,0 +1,141 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.test; + +import org.apache.http.client.fluent.Request; +import org.elasticsearch.packaging.util.Distribution; +import org.elasticsearch.packaging.util.FileUtils; +import org.elasticsearch.packaging.util.ServerUtils; +import org.elasticsearch.packaging.util.Shell; +import org.junit.Before; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.elasticsearch.packaging.util.FileUtils.append; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.collection.IsMapContaining.hasKey; +import static org.junit.Assume.assumeTrue; + +public class PasswordToolsTests extends PackagingTestCase { + + private static final Pattern USERPASS_REGEX = Pattern.compile("PASSWORD (\\w+) = ([^\\s]+)"); + private static final String BOOTSTRAP_PASSWORD = "myS3curepass"; + + @Before + public void filterDistros() { + assumeTrue("only default distro", distribution.flavor == Distribution.Flavor.DEFAULT); + assumeTrue("no docker", distribution.packaging != Distribution.Packaging.DOCKER); + } + + public void test010Install() throws Exception { + install(); + append(installation.config("elasticsearch.yml"), + "xpack.license.self_generated.type: trial\n" + + "xpack.security.enabled: true"); + } + + public void test20GeneratePasswords() throws Exception { + assertWhileRunning(() -> { + Shell.Result result = installation.executables().elasticsearchSetupPasswords.run(sh, "auto --batch", null); + Map userpasses = parseUsersAndPasswords(result.stdout); + for (Map.Entry userpass : userpasses.entrySet()) { + String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue()); + assertThat(response, containsString("You Know, for Search")); + } + }); + } + + public void test30AddBootstrapPassword() throws Exception { + + try (Stream dataFiles = Files.list(installation.data)) { + // delete each dir under data, not data itself + dataFiles.forEach(file -> { + if (distribution.platform != Distribution.Platform.WINDOWS) { + FileUtils.rm(file); + return; + } + // HACK: windows asynchronously releases file locks after processes exit. Unfortunately there is no clear way to wait on + // those locks being released. We might be able to use `openfiles /query`, but that requires modifying global settings + // in our windows images with `openfiles /local on` (which requires a restart, thus needs to be baked into the images). + // The following sleep allows time for windows to release the data file locks from Elasticsearch which was stopped in the + // previous test. + int retries = 30; + Exception failure = null; + while (retries-- > 0) { + try { + FileUtils.rm(file); + return; + } catch (Exception e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + try { + Thread.sleep(1000); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return; + } + } + } + throw new RuntimeException("failed to delete " + file, failure); + }); + } + + installation.executables().elasticsearchKeystore.run(sh, "add --stdin bootstrap.password", BOOTSTRAP_PASSWORD); + + assertWhileRunning(() -> { + String response = ServerUtils.makeRequest( + Request.Get("http://localhost:9200/_cluster/health?wait_for_status=green&timeout=180s"), + "elastic", BOOTSTRAP_PASSWORD); + assertThat(response, containsString("\"status\":\"green\"")); + }); + } + + public void test40GeneratePasswordsBootstrapAlreadySet() throws Exception { + assertWhileRunning(() -> { + + Shell.Result result = installation.executables().elasticsearchSetupPasswords.run(sh, "auto --batch", null); + Map userpasses = parseUsersAndPasswords(result.stdout); + assertThat(userpasses, hasKey("elastic")); + for (Map.Entry userpass : userpasses.entrySet()) { + String response = ServerUtils.makeRequest(Request.Get("http://localhost:9200"), userpass.getKey(), userpass.getValue()); + assertThat(response, containsString("You Know, for Search")); + } + }); + } + + private Map parseUsersAndPasswords(String output) { + Matcher matcher = USERPASS_REGEX.matcher(output); + assertNotNull(matcher); + Map userpases = new HashMap<>(); + while (matcher.find()) { + userpases.put(matcher.group(1), matcher.group(2)); + } + return userpases; + } +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/SqlCliTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/SqlCliTests.java new file mode 100644 index 0000000000000..62a00aab59d22 --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/SqlCliTests.java @@ -0,0 +1,44 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.test; + +import org.elasticsearch.packaging.util.Distribution; +import org.elasticsearch.packaging.util.Shell; +import org.junit.Before; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assume.assumeTrue; + +public class SqlCliTests extends PackagingTestCase { + @Before + public void filterDistros() { + assumeTrue("only default distro", distribution.flavor == Distribution.Flavor.DEFAULT); + assumeTrue("no docker", distribution.packaging != Distribution.Packaging.DOCKER); + } + + public void test010Install() throws Exception { + install(); + } + + public void test020Help() throws Exception { + Shell.Result result = installation.executables().elasticsearchSqlCli.run(sh, "--help"); + assertThat(result.stdout, containsString("Elasticsearch SQL CLI")); + } +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java index 56995bab1de0c..4303ca3bb6dd1 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java @@ -111,7 +111,7 @@ public static Installation installArchive(Distribution distribution, Path fullIn sh.chown(fullInstallPath); - return Installation.ofArchive(fullInstallPath); + return Installation.ofArchive(distribution, fullInstallPath); } private static void setupArchiveUsersLinux(Path installPath) { diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index 8489ef42ea83c..8d5b7ce12c72f 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -24,6 +24,8 @@ import org.elasticsearch.common.CheckedRunnable; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.PosixFileAttributes; import java.nio.file.attribute.PosixFilePermission; import java.util.ArrayList; import java.util.List; @@ -35,6 +37,7 @@ import static org.elasticsearch.packaging.util.FileMatcher.p644; import static org.elasticsearch.packaging.util.FileMatcher.p660; import static org.elasticsearch.packaging.util.FileMatcher.p755; +import static org.elasticsearch.packaging.util.FileMatcher.p770; import static org.elasticsearch.packaging.util.FileMatcher.p775; import static org.elasticsearch.packaging.util.FileUtils.getCurrentVersion; import static org.hamcrest.CoreMatchers.containsString; @@ -96,7 +99,7 @@ public static Installation runContainer(Distribution distribution, Map args = new ArrayList<>(); + + args.add("docker run"); + + // Don't leave orphaned containers + args.add("--rm"); + + // Mount localPath to a known location inside the container, so that we can execute shell commands on it later + args.add("--volume \"" + localPath.getParent() + ":" + containerPath.getParent() + "\""); + + // Use a lightweight musl libc based small image + args.add("alpine"); + + // And run inline commands via the POSIX shell + args.add("/bin/sh -c \"" + shellCmd + "\""); + + final String command = String.join(" ", args); + logger.info("Running command: " + command); + sh.run(command); + } + + /** + * Create a directory with specified uid/gid using Docker backed privilege escalation. + * @param localPath The path to the directory to create. + * @param uid The numeric id for localPath + * @param gid The numeric id for localPath + */ + public static void mkDirWithPrivilegeEscalation(Path localPath, int uid, int gid) { + final Path containerBasePath = Paths.get("/mount"); + final Path containerPath = containerBasePath.resolve(Paths.get("/").relativize(localPath)); + final List args = new ArrayList<>(); + + args.add("mkdir " + containerPath.toAbsolutePath()); + args.add("&&"); + args.add("chown " + uid + ":" + gid + " " + containerPath.toAbsolutePath()); + args.add("&&"); + args.add("chmod 0770 " + containerPath.toAbsolutePath()); + final String command = String.join(" ", args); + executePrivilegeEscalatedShellCmd(command, localPath, containerPath); + + final PosixFileAttributes dirAttributes = FileUtils.getPosixFileAttributes(localPath); + final Map numericPathOwnership = FileUtils.getNumericUnixPathOwnership(localPath); + assertEquals(localPath + " has wrong uid", numericPathOwnership.get("uid").intValue(), uid); + assertEquals(localPath + " has wrong gid", numericPathOwnership.get("gid").intValue(), gid); + assertEquals(localPath + " has wrong permissions", dirAttributes.permissions(), p770); + } + + /** + * Delete a directory using Docker backed privilege escalation. + * @param localPath The path to the directory to delete. + */ + public static void rmDirWithPrivilegeEscalation(Path localPath) { + final Path containerBasePath = Paths.get("/mount"); + final Path containerPath = containerBasePath.resolve(Paths.get("/").relativize(localPath)); + final List args = new ArrayList<>(); + + args.add("cd " + containerBasePath.toAbsolutePath()); + args.add("&&"); + args.add("rm -rf " + localPath.getFileName()); + final String command = String.join(" ", args); + executePrivilegeEscalatedShellCmd(command, localPath, containerPath); + } + /** * Checks that the specified path's permissions and ownership match those specified. */ diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java index ba3671500e931..57c74a8c68e01 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java @@ -46,6 +46,7 @@ public class FileMatcher extends TypeSafeMatcher { public enum Fileness { File, Directory } public static final Set p775 = fromString("rwxrwxr-x"); + public static final Set p770 = fromString("rwxrwx---"); public static final Set p755 = fromString("rwxr-xr-x"); public static final Set p750 = fromString("rwxr-x---"); public static final Set p660 = fromString("rw-rw----"); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java index 16824ba4ae413..efd2c85bf9e3a 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java @@ -33,6 +33,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; @@ -42,7 +43,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.StringJoiner; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -240,6 +243,23 @@ public static PosixFileAttributes getPosixFileAttributes(Path path) { } } + /** + * Gets numeric ownership attributes that are supported by Unix filesystems + * @return a Map of the uid/gid integer values + */ + public static Map getNumericUnixPathOwnership(Path path) { + Map numericPathOwnership = new HashMap<>(); + + try { + numericPathOwnership.put("uid", (int) Files.getAttribute(path, "unix:uid", LinkOption.NOFOLLOW_LINKS)); + numericPathOwnership.put("gid", (int) Files.getAttribute(path, "unix:gid", LinkOption.NOFOLLOW_LINKS)); + } catch (IOException e) { + throw new RuntimeException(e); + } + return numericPathOwnership; + } + + // vagrant creates /tmp for us in windows so we use that to avoid long paths public static Path getTempDir() { return Paths.get("/tmp"); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java index c5fdf0106df29..1f3da0b12663f 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java @@ -27,6 +27,12 @@ */ public class Installation { + // in the future we'll run as a role user on Windows + public static final String ARCHIVE_OWNER = Platforms.WINDOWS + ? System.getenv("username") + : "elasticsearch"; + + public final Distribution distribution; public final Path home; public final Path bin; // this isn't a first-class installation feature but we include it for convenience public final Path lib; // same @@ -39,7 +45,9 @@ public class Installation { public final Path pidDir; public final Path envFile; - public Installation(Path home, Path config, Path data, Path logs, Path plugins, Path modules, Path pidDir, Path envFile) { + private Installation(Distribution distribution, Path home, Path config, Path data, Path logs, + Path plugins, Path modules, Path pidDir, Path envFile) { + this.distribution = distribution; this.home = home; this.bin = home.resolve("bin"); this.lib = home.resolve("lib"); @@ -53,8 +61,9 @@ public Installation(Path home, Path config, Path data, Path logs, Path plugins, this.envFile = envFile; } - public static Installation ofArchive(Path home) { + public static Installation ofArchive(Distribution distribution, Path home) { return new Installation( + distribution, home, home.resolve("config"), home.resolve("data"), @@ -66,13 +75,14 @@ public static Installation ofArchive(Path home) { ); } - public static Installation ofPackage(Distribution.Packaging packaging) { + public static Installation ofPackage(Distribution distribution) { - final Path envFile = (packaging == Distribution.Packaging.RPM) + final Path envFile = (distribution.packaging == Distribution.Packaging.RPM) ? Paths.get("/etc/sysconfig/elasticsearch") : Paths.get("/etc/default/elasticsearch"); return new Installation( + distribution, Paths.get("/usr/share/elasticsearch"), Paths.get("/etc/elasticsearch"), Paths.get("/var/lib/elasticsearch"), @@ -84,9 +94,10 @@ public static Installation ofPackage(Distribution.Packaging packaging) { ); } - public static Installation ofContainer() { + public static Installation ofContainer(Distribution distribution) { String root = "/usr/share/elasticsearch"; return new Installation( + distribution, Paths.get(root), Paths.get(root + "/config"), Paths.get(root + "/data"), @@ -110,23 +121,48 @@ public Executables executables() { return new Executables(); } - public class Executables { + public class Executable { + public final Path path; - public final Path elasticsearch = platformExecutable("elasticsearch"); - public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin"); - public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore"); - public final Path elasticsearchCertutil = platformExecutable("elasticsearch-certutil"); - public final Path elasticsearchShard = platformExecutable("elasticsearch-shard"); - public final Path elasticsearchNode = platformExecutable("elasticsearch-node"); - public final Path elasticsearchSetupPasswords = platformExecutable("elasticsearch-setup-passwords"); - public final Path elasticsearchSyskeygen = platformExecutable("elasticsearch-syskeygen"); - public final Path elasticsearchUsers = platformExecutable("elasticsearch-users"); - - private Path platformExecutable(String name) { + private Executable(String name) { final String platformExecutableName = Platforms.WINDOWS ? name + ".bat" : name; - return bin(platformExecutableName); + this.path = bin(platformExecutableName); + } + + @Override + public String toString() { + return path.toString(); + } + + public Shell.Result run(Shell sh, String args) { + return run(sh, args, null); } + + public Shell.Result run(Shell sh, String args, String input) { + String command = path + " " + args; + if (distribution.isArchive() && distribution.platform != Distribution.Platform.WINDOWS) { + command = "sudo -E -u " + ARCHIVE_OWNER + " " + command; + } + if (input != null) { + command = "echo \"" + input + "\" | " + command; + } + return sh.run(command); + } + } + + public class Executables { + + public final Executable elasticsearch = new Executable("elasticsearch"); + public final Executable elasticsearchPlugin = new Executable("elasticsearch-plugin"); + public final Executable elasticsearchKeystore = new Executable("elasticsearch-keystore"); + public final Executable elasticsearchCertutil = new Executable("elasticsearch-certutil"); + public final Executable elasticsearchShard = new Executable("elasticsearch-shard"); + public final Executable elasticsearchNode = new Executable("elasticsearch-node"); + public final Executable elasticsearchSetupPasswords = new Executable("elasticsearch-setup-passwords"); + public final Executable elasticsearchSqlCli= new Executable("elasticsearch-sql-cli"); + public final Executable elasticsearchSyskeygen = new Executable("elasticsearch-syskeygen"); + public final Executable elasticsearchUsers = new Executable("elasticsearch-users"); } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java index 065e77a239bd5..723625f1bb8bc 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -106,7 +106,7 @@ public static Installation installPackage(Distribution distribution) throws IOEx throw new RuntimeException("Installing distribution " + distribution + " failed: " + result); } - Installation installation = Installation.ofPackage(distribution.packaging); + Installation installation = Installation.ofPackage(distribution); if (distribution.hasJdk == false) { Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8), diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index 1a4113738d2ed..40aee0a014175 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -29,6 +29,11 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.net.InetAddress; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -39,13 +44,32 @@ public class ServerUtils { private static final Logger logger = LogManager.getLogger(ServerUtils.class); + private static String SECURITY_ENABLED = "xpack.security.enabled: true"; + // generous timeout as nested virtualization can be quite slow ... private static final long waitTime = TimeUnit.MINUTES.toMillis(3); private static final long timeoutLength = TimeUnit.SECONDS.toMillis(30); private static final long requestInterval = TimeUnit.SECONDS.toMillis(5); public static void waitForElasticsearch(Installation installation) throws IOException { - waitForElasticsearch("green", null, installation, null, null); + boolean securityEnabled = false; + + // TODO: need a way to check if docker has security enabled, the yml config is not bind mounted so can't look from here + if (installation.distribution.packaging != Distribution.Packaging.DOCKER) { + Path configFilePath = installation.config("elasticsearch.yml"); + // this is fragile, but currently doesn't deviate from a single line enablement and not worth the parsing effort + String configFile = Files.readString(configFilePath, StandardCharsets.UTF_8); + securityEnabled = configFile.contains(SECURITY_ENABLED); + } + + if (securityEnabled) { + // with security enabled, we may or may not have setup a user/pass, so we use a more generic port being available check. + // this isn't as good as a health check, but long term all this waiting should go away when node startup does not + // make the http port available until the system is really ready to serve requests + waitForXpack(); + } else { + waitForElasticsearch("green", null, installation, null, null); + } } /** @@ -68,6 +92,26 @@ private static HttpResponse execute(Request request, String username, String pas return executor.execute(request).returnResponse(); } + // polls every second for Elasticsearch to be running on 9200 + private static void waitForXpack() { + int retries = 60; + while (retries > 0) { + retries -= 1; + try (Socket s = new Socket(InetAddress.getLoopbackAddress(), 9200)) { + return; + } catch (IOException e) { + // ignore, only want to establish a connection + } + try { + Thread.sleep(1000); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + return; + } + } + throw new RuntimeException("Elasticsearch (with x-pack) did not start"); + } + public static void waitForElasticsearch( String status, String index, diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index 375ba12a35621..fa86389f0db2a 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -42,7 +42,7 @@ rest_total_hits_as_int: true index: queries body: - sort: _id + sort: id query: percolate: field: query diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml index 2672cee7cc78a..e1ffcea930a42 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/10_basic.yml @@ -66,6 +66,8 @@ body: mappings: properties: + id: + type: keyword query: type: percolator field1: @@ -80,6 +82,7 @@ index: queries id: q1 body: + id: q1 query: term: field1: value @@ -89,6 +92,7 @@ index: queries id: q2 body: + id: q2 query: bool: must: @@ -102,6 +106,7 @@ index: queries id: q3 body: + id: q3 query: bool: minimum_should_match: 2 @@ -133,7 +138,7 @@ rest_total_hits_as_int: true index: queries body: - sort: _id + sort: id query: percolate: field: query @@ -149,7 +154,7 @@ rest_total_hits_as_int: true index: queries body: - sort: _id + sort: id query: percolate: field: query diff --git a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml index 78a4aac867d8f..4368ffd602f58 100644 --- a/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml +++ b/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/10_basic.yml @@ -30,6 +30,7 @@ id: q4 refresh: true body: + id: q4 query: bool: minimum_should_match: 2 @@ -57,7 +58,7 @@ rest_total_hits_as_int: true index: queries body: - sort: _id + sort: id query: percolate: field: query diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json index abf9c2d330829..5c54c86906e69 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json @@ -89,8 +89,7 @@ "options":[ "internal", "external", - "external_gte", - "force" + "external_gte" ], "description":"Specific version type" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json index fd221b474a070..a8e6b6786ca90 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json @@ -67,8 +67,7 @@ "options":[ "internal", "external", - "external_gte", - "force" + "external_gte" ], "description":"Specific version type" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json b/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json index 143ee406025ce..3ae8d60ee6ede 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json @@ -88,8 +88,7 @@ "options":[ "internal", "external", - "external_gte", - "force" + "external_gte" ], "description":"Specific version type" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json index 2ce77f17aff10..36d08c0313bde 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json @@ -67,8 +67,7 @@ "options":[ "internal", "external", - "external_gte", - "force" + "external_gte" ], "description":"Specific version type" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json index ad79678388590..d99a188242480 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json @@ -63,8 +63,7 @@ "options":[ "internal", "external", - "external_gte", - "force" + "external_gte" ], "description":"Specific version type" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json b/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json index d5fc7371e0898..0afe5a316b285 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/mtermvectors.json @@ -84,8 +84,7 @@ "options":[ "internal", "external", - "external_gte", - "force" + "external_gte" ], "description":"Specific version type" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json b/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json index b6cb3663c2df2..d9ba7c0b36cc5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/termvectors.json @@ -90,8 +90,7 @@ "options":[ "internal", "external", - "external_gte", - "force" + "external_gte" ], "description":"Specific version type" } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/20_missing_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/20_missing_field.yml index 77b795686db4e..408d0742ab8a6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/20_missing_field.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.get_field_mapping/20_missing_field.yml @@ -1,5 +1,5 @@ --- -"Return empty object if field doesn't exist, but type and index do": +"Return empty object if field doesn't exist, but index does": - do: indices.create: @@ -17,3 +17,4 @@ fields: not_existent - match: { 'test_index.mappings': {}} + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index ff001056957b7..98c8bdff66fc6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -6,19 +6,19 @@ setup: index: index: test id: 1 - body: { foo: bar, age: 18 } + body: { id: 1, foo: bar, age: 18 } - do: index: index: test id: 42 - body: { foo: bar, age: 18 } + body: { id: 42, foo: bar, age: 18 } - do: index: index: test id: 172 - body: { foo: bar, age: 24 } + body: { id: 172, foo: bar, age: 24 } - do: indices.refresh: @@ -36,13 +36,13 @@ setup: query: match: foo: bar - sort: [{ age: desc }, { _id: desc }] + sort: [{ age: desc }, { id: desc }] - match: {hits.total: 3 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - match: {hits.hits.0._id: "172" } - - match: {hits.hits.0.sort: [24, "172"] } + - match: {hits.hits.0.sort: [24, 172] } - do: search: @@ -53,14 +53,14 @@ setup: query: match: foo: bar - sort: [{ age: desc }, { _id: desc }] - search_after: [24, "172"] + sort: [{ age: desc }, { id: desc }] + search_after: [24, 172] - match: {hits.total: 3 } - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - match: {hits.hits.0._id: "42" } - - match: {hits.hits.0.sort: [18, "42"] } + - match: {hits.hits.0.sort: [18, 42] } - do: search: @@ -71,14 +71,14 @@ setup: query: match: foo: bar - sort: [ { age: desc }, { _id: desc } ] - search_after: [18, "42"] + sort: [ { age: desc }, { id: desc } ] + search_after: [18, 42] - match: {hits.total: 3} - length: {hits.hits: 1 } - match: {hits.hits.0._index: test } - match: {hits.hits.0._id: "1" } - - match: {hits.hits.0.sort: [18, "1"] } + - match: {hits.hits.0.sort: [18, 1] } - do: search: @@ -89,8 +89,8 @@ setup: query: match: foo: bar - sort: [{ age: desc }, { _id: desc } ] - search_after: [18, "1"] + sort: [{ age: desc }, { id: desc } ] + search_after: [18, 1] - match: {hits.total: 3} - length: {hits.hits: 0 } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java index 576d3812c0cf4..ea0e14fc05c49 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsIndexRequest.java @@ -19,10 +19,12 @@ package org.elasticsearch.action.admin.indices.mapping.get; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.single.shard.SingleShardRequest; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -30,26 +32,26 @@ public class GetFieldMappingsIndexRequest extends SingleShardRequest { - private final boolean probablySingleFieldRequest; private final boolean includeDefaults; private final String[] fields; - private final String[] types; - private OriginalIndices originalIndices; + private final OriginalIndices originalIndices; GetFieldMappingsIndexRequest(StreamInput in) throws IOException { super(in); - types = in.readStringArray(); + if (in.getVersion().before(Version.V_8_0_0)) { + in.readStringArray(); // former types array + } fields = in.readStringArray(); includeDefaults = in.readBoolean(); - probablySingleFieldRequest = in.readBoolean(); + if (in.getVersion().before(Version.V_8_0_0)) { + in.readBoolean(); // former probablySingleField boolean + } originalIndices = OriginalIndices.readOriginalIndices(in); } - GetFieldMappingsIndexRequest(GetFieldMappingsRequest other, String index, boolean probablySingleFieldRequest) { - this.probablySingleFieldRequest = probablySingleFieldRequest; + GetFieldMappingsIndexRequest(GetFieldMappingsRequest other, String index) { this.includeDefaults = other.includeDefaults(); - this.types = other.types(); this.fields = other.fields(); assert index != null; this.index(index); @@ -61,18 +63,10 @@ public ActionRequestValidationException validate() { return null; } - public String[] types() { - return types; - } - public String[] fields() { return fields; } - public boolean probablySingleFieldRequest() { - return probablySingleFieldRequest; - } - public boolean includeDefaults() { return includeDefaults; } @@ -90,10 +84,14 @@ public IndicesOptions indicesOptions() { @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); - out.writeStringArray(types); + if (out.getVersion().before(Version.V_8_0_0)) { + out.writeStringArray(Strings.EMPTY_ARRAY); + } out.writeStringArray(fields); out.writeBoolean(includeDefaults); - out.writeBoolean(probablySingleFieldRequest); + if (out.getVersion().before(Version.V_8_0_0)) { + out.writeBoolean(false); + } OriginalIndices.writeOriginalIndices(originalIndices, out); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java index 7442fe2dd3597..af0c821fdd8e3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequest.java @@ -19,6 +19,7 @@ package org.elasticsearch.action.admin.indices.mapping.get; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; @@ -28,6 +29,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.IOException; +import java.util.Arrays; /** * Request the mappings of specific fields @@ -44,7 +46,6 @@ public class GetFieldMappingsRequest extends ActionRequest implements IndicesReq private boolean includeDefaults = false; private String[] indices = Strings.EMPTY_ARRAY; - private String[] types = Strings.EMPTY_ARRAY; private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpen(); @@ -53,7 +54,12 @@ public GetFieldMappingsRequest() {} public GetFieldMappingsRequest(StreamInput in) throws IOException { super(in); indices = in.readStringArray(); - types = in.readStringArray(); + if (in.getVersion().before(Version.V_8_0_0)) { + String[] types = in.readStringArray(); + if (types != Strings.EMPTY_ARRAY) { + throw new IllegalArgumentException("Expected empty type array but received [" + Arrays.toString(types) + "]"); + } + } indicesOptions = IndicesOptions.readIndicesOptions(in); local = in.readBoolean(); fields = in.readStringArray(); @@ -79,11 +85,6 @@ public GetFieldMappingsRequest indices(String... indices) { return this; } - public GetFieldMappingsRequest types(String... types) { - this.types = types; - return this; - } - public GetFieldMappingsRequest indicesOptions(IndicesOptions indicesOptions) { this.indicesOptions = indicesOptions; return this; @@ -94,10 +95,6 @@ public String[] indices() { return indices; } - public String[] types() { - return types; - } - @Override public IndicesOptions indicesOptions() { return indicesOptions; @@ -132,7 +129,9 @@ public ActionRequestValidationException validate() { public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringArray(indices); - out.writeStringArray(types); + if (out.getVersion().before(Version.V_8_0_0)) { + out.writeStringArray(Strings.EMPTY_ARRAY); + } indicesOptions.writeIndicesOptions(out); out.writeBoolean(local); out.writeStringArray(fields); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java index cbd0539c24485..ae0c741d242d2 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsRequestBuilder.java @@ -42,16 +42,6 @@ public GetFieldMappingsRequestBuilder addIndices(String... indices) { return this; } - public GetFieldMappingsRequestBuilder setTypes(String... types) { - request.types(types); - return this; - } - - public GetFieldMappingsRequestBuilder addTypes(String... types) { - request.types(ArrayUtils.concat(request.types(), types)); - return this; - } - public GetFieldMappingsRequestBuilder setIndicesOptions(IndicesOptions indicesOptions) { request.indicesOptions(indicesOptions); return this; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java index 3cc5e92ec81f3..bd4fbc5f82af5 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java @@ -19,9 +19,9 @@ package org.elasticsearch.action.admin.indices.mapping.get; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; import java.io.IOException; import java.io.InputStream; @@ -51,38 +52,35 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte private static final ParseField MAPPINGS = new ParseField("mappings"); - // TODO remove the middle `type` level of this - private final Map>> mappings; + private final Map> mappings; - GetFieldMappingsResponse(Map>> mappings) { + GetFieldMappingsResponse(Map> mappings) { this.mappings = mappings; } GetFieldMappingsResponse(StreamInput in) throws IOException { super(in); int size = in.readVInt(); - Map>> indexMapBuilder = new HashMap<>(size); + Map> indexMapBuilder = new HashMap<>(size); for (int i = 0; i < size; i++) { String index = in.readString(); - int typesSize = in.readVInt(); - Map> typeMapBuilder = new HashMap<>(typesSize); - for (int j = 0; j < typesSize; j++) { - String type = in.readString(); - int fieldSize = in.readVInt(); - Map fieldMapBuilder = new HashMap<>(fieldSize); - for (int k = 0; k < fieldSize; k++) { - fieldMapBuilder.put(in.readString(), new FieldMappingMetaData(in.readString(), in.readBytesReference())); - } - typeMapBuilder.put(type, unmodifiableMap(fieldMapBuilder)); + if (in.getVersion().before(Version.V_8_0_0)) { + int typesSize = in.readVInt(); + assert typesSize == 1; + in.readString(); // type + } + int fieldSize = in.readVInt(); + Map fieldMapBuilder = new HashMap<>(fieldSize); + for (int k = 0; k < fieldSize; k++) { + fieldMapBuilder.put(in.readString(), new FieldMappingMetaData(in.readString(), in.readBytesReference())); } - indexMapBuilder.put(index, unmodifiableMap(typeMapBuilder)); + indexMapBuilder.put(index, unmodifiableMap(fieldMapBuilder)); } mappings = unmodifiableMap(indexMapBuilder); - } - /** returns the retrieved field mapping. The return map keys are index, type, field (as specified in the request). */ - public Map>> mappings() { + /** returns the retrieved field mapping. The return map keys are index, field (as specified in the request). */ + public Map> mappings() { return mappings; } @@ -92,32 +90,23 @@ public Map>> mappings() { * @param field field name as specified in the {@link GetFieldMappingsRequest} * @return FieldMappingMetaData for the requested field or null if not found. */ - public FieldMappingMetaData fieldMappings(String index, String type, String field) { - Map> indexMapping = mappings.get(index); + public FieldMappingMetaData fieldMappings(String index, String field) { + Map indexMapping = mappings.get(index); if (indexMapping == null) { return null; } - Map typeMapping = indexMapping.get(type); - if (typeMapping == null) { - return null; - } - return typeMapping.get(field); + return indexMapping.get(field); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - for (Map.Entry>> indexEntry : mappings.entrySet()) { + builder.startObject(); + for (Map.Entry> indexEntry : mappings.entrySet()) { builder.startObject(indexEntry.getKey()); builder.startObject(MAPPINGS.getPreferredName()); - Map mappings = null; - for (Map.Entry> typeEntry : indexEntry.getValue().entrySet()) { - assert mappings == null; - mappings = typeEntry.getValue(); - } - if (mappings != null) { - addFieldMappingsToBuilder(builder, params, mappings); + if (indexEntry.getValue() != null) { + addFieldMappingsToBuilder(builder, params, indexEntry.getValue()); } builder.endObject(); @@ -138,7 +127,6 @@ private void addFieldMappingsToBuilder(XContentBuilder builder, } public static class FieldMappingMetaData implements ToXContentFragment { - public static final FieldMappingMetaData NULL = new FieldMappingMetaData("", BytesArray.EMPTY); private static final ParseField FULL_NAME = new ParseField("full_name"); private static final ParseField MAPPING = new ParseField("mapping"); @@ -165,10 +153,6 @@ public Map sourceAsMap() { return XContentHelper.convertToMap(source, true, XContentType.JSON).v2(); } - public boolean isNull() { - return NULL.fullName().equals(fullName) && NULL.source.length() == source.length(); - } - //pkg-private for testing BytesReference getSource() { return source; @@ -210,18 +194,18 @@ public int hashCode() { @Override public void writeTo(StreamOutput out) throws IOException { out.writeVInt(mappings.size()); - for (Map.Entry>> indexEntry : mappings.entrySet()) { + for (Map.Entry> indexEntry : mappings.entrySet()) { out.writeString(indexEntry.getKey()); + if (out.getVersion().before(Version.V_8_0_0)) { + out.writeVInt(1); + out.writeString(MapperService.SINGLE_MAPPING_NAME); + } out.writeVInt(indexEntry.getValue().size()); - for (Map.Entry> typeEntry : indexEntry.getValue().entrySet()) { - out.writeString(typeEntry.getKey()); - out.writeVInt(typeEntry.getValue().size()); - for (Map.Entry fieldEntry : typeEntry.getValue().entrySet()) { - out.writeString(fieldEntry.getKey()); - FieldMappingMetaData fieldMapping = fieldEntry.getValue(); - out.writeString(fieldMapping.fullName()); - out.writeBytesReference(fieldMapping.source); - } + for (Map.Entry fieldEntry : indexEntry.getValue().entrySet()) { + out.writeString(fieldEntry.getKey()); + FieldMappingMetaData fieldMapping = fieldEntry.getValue(); + out.writeString(fieldMapping.fullName()); + out.writeBytesReference(fieldMapping.source); } } } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java index 30b939d627e38..61e84147e3d53 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/GetMappingsResponse.java @@ -99,13 +99,13 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { for (final ObjectObjectCursor indexEntry : getMappings()) { + builder.startObject(indexEntry.key); if (indexEntry.value != null) { - builder.startObject(indexEntry.key); builder.field(MAPPINGS.getPreferredName(), indexEntry.value.sourceAsMap()); - builder.endObject(); } else { builder.startObject(MAPPINGS.getPreferredName()).endObject(); } + builder.endObject(); } return builder; } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java index c95435834b499..9f2124f390e3a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsAction.java @@ -65,9 +65,8 @@ protected void doExecute(Task task, GetFieldMappingsRequest request, final Actio if (concreteIndices.length == 0) { listener.onResponse(new GetFieldMappingsResponse(emptyMap())); } else { - boolean probablySingleFieldRequest = concreteIndices.length == 1 && request.types().length == 1 && request.fields().length == 1; for (final String index : concreteIndices) { - GetFieldMappingsIndexRequest shardRequest = new GetFieldMappingsIndexRequest(request, index, probablySingleFieldRequest); + GetFieldMappingsIndexRequest shardRequest = new GetFieldMappingsIndexRequest(request, index); client.executeLocally(TransportGetFieldMappingsIndexAction.TYPE, shardRequest, new ActionListener<>() { @Override @@ -92,7 +91,7 @@ public void onFailure(Exception e) { } private GetFieldMappingsResponse merge(AtomicReferenceArray indexResponses) { - Map>> mergedResponses = new HashMap<>(); + Map> mergedResponses = new HashMap<>(); for (int i = 0; i < indexResponses.length(); i++) { Object element = indexResponses.get(i); if (element instanceof GetFieldMappingsResponse) { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java index 842f00d59c6d0..c7fe23b38d0e7 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java @@ -44,12 +44,10 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; -import org.elasticsearch.indices.TypeMissingException; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import java.io.IOException; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -99,30 +97,9 @@ protected GetFieldMappingsResponse shardOperation(final GetFieldMappingsIndexReq Predicate metadataFieldPredicate = (f) -> indicesService.isMetaDataField(indexCreatedVersion, f); Predicate fieldPredicate = metadataFieldPredicate.or(indicesService.getFieldFilter().apply(shardId.getIndexName())); - DocumentMapper mapper = indexService.mapperService().documentMapper(); - Collection typeIntersection; - if (request.types().length == 0) { - typeIntersection = mapper == null - ? Collections.emptySet() - : Collections.singleton(mapper.type()); - } else { - typeIntersection = mapper != null && Regex.simpleMatch(request.types(), mapper.type()) - ? Collections.singleton(mapper.type()) - : Collections.emptySet(); - if (typeIntersection.isEmpty()) { - throw new TypeMissingException(shardId.getIndex(), request.types()); - } - } - - Map> typeMappings = new HashMap<>(); - for (String type : typeIntersection) { - DocumentMapper documentMapper = indexService.mapperService().documentMapper(); - Map fieldMapping = findFieldMappingsByType(fieldPredicate, documentMapper, request); - if (!fieldMapping.isEmpty()) { - typeMappings.put(type, fieldMapping); - } - } - return new GetFieldMappingsResponse(singletonMap(shardId.getIndexName(), Collections.unmodifiableMap(typeMappings))); + DocumentMapper documentMapper = indexService.mapperService().documentMapper(); + Map fieldMapping = findFieldMappings(fieldPredicate, documentMapper, request); + return new GetFieldMappingsResponse(singletonMap(shardId.getIndexName(), fieldMapping)); } @Override @@ -172,9 +149,12 @@ public Boolean paramAsBoolean(String key, Boolean defaultValue) { } }; - private static Map findFieldMappingsByType(Predicate fieldPredicate, + private static Map findFieldMappings(Predicate fieldPredicate, DocumentMapper documentMapper, GetFieldMappingsIndexRequest request) { + if (documentMapper == null) { + return Collections.emptyMap(); + } Map fieldMappings = new HashMap<>(); final DocumentFieldMappers allFieldMappers = documentMapper.mappers(); for (String field : request.fields()) { @@ -194,8 +174,6 @@ private static Map findFieldMappingsByType(Predica Mapper fieldMapper = allFieldMappers.getMapper(field); if (fieldMapper != null) { addFieldMapper(fieldPredicate, field, fieldMapper, fieldMappings, request.includeDefaults()); - } else if (request.probablySingleFieldRequest()) { - fieldMappings.put(field, FieldMappingMetaData.NULL); } } } diff --git a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java index 070e99cc5c775..79de0d0c2a7fd 100644 --- a/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java +++ b/server/src/main/java/org/elasticsearch/action/ingest/SimulateExecutionService.java @@ -54,7 +54,7 @@ void executeDocument(Pipeline pipeline, IngestDocument ingestDocument, boolean v handler.accept(new SimulateDocumentVerboseResult(processorResultList), e); }); } else { - pipeline.execute(ingestDocument, (result, e) -> { + ingestDocument.executePipeline(pipeline, (result, e) -> { if (e == null) { handler.accept(new SimulateDocumentBaseResult(result), null); } else { diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java index f2fe9a84e06a7..a401123196662 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateHelper.java @@ -99,8 +99,7 @@ protected Result prepare(ShardId shardId, UpdateRequest request, final GetResult * Execute a scripted upsert, where there is an existing upsert document and a script to be executed. The script is executed and a new * Tuple of operation and updated {@code _source} is returned. */ - Tuple> executeScriptedUpsert(IndexRequest upsert, Script script, LongSupplier nowInMillis) { - Map upsertDoc = upsert.sourceAsMap(); + Tuple> executeScriptedUpsert(Map upsertDoc, Script script, LongSupplier nowInMillis) { Map ctx = new HashMap<>(3); // Tell the script that this is a create and not an update ctx.put(ContextFields.OP, UpdateOpType.CREATE.toString()); @@ -133,11 +132,11 @@ Result prepareUpsert(ShardId shardId, UpdateRequest request, final GetResult get if (request.scriptedUpsert() && request.script() != null) { // Run the script to perform the create logic IndexRequest upsert = request.upsertRequest(); - Tuple> upsertResult = executeScriptedUpsert(upsert, request.script, nowInMillis); + Tuple> upsertResult = executeScriptedUpsert(upsert.sourceAsMap(), request.script, + nowInMillis); switch (upsertResult.v1()) { case CREATE: - // Update the index request with the new "_source" - indexRequest.source(upsertResult.v2()); + indexRequest = Requests.indexRequest(request.index()).source(upsertResult.v2()); break; case NONE: UpdateResponse update = new UpdateResponse(shardId, getResult.getId(), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java index 3ad8f099c69f0..52ee0be49ef65 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexUpgradeService.java @@ -180,7 +180,7 @@ public Set> entrySet() { try (IndexAnalyzers fakeIndexAnalzyers = new IndexAnalyzers(analyzerMap, analyzerMap, analyzerMap)) { MapperService mapperService = new MapperService(indexSettings, fakeIndexAnalzyers, xContentRegistry, similarityService, - mapperRegistry, () -> null); + mapperRegistry, () -> null, () -> false); mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY); } } catch (Exception ex) { diff --git a/server/src/main/java/org/elasticsearch/common/io/Streams.java b/server/src/main/java/org/elasticsearch/common/io/Streams.java index 4a8f2f5de5b74..222f94e65ef6a 100644 --- a/server/src/main/java/org/elasticsearch/common/io/Streams.java +++ b/server/src/main/java/org/elasticsearch/common/io/Streams.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import java.io.BufferedReader; +import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -219,6 +220,21 @@ public static void readAllLines(InputStream input, Consumer consumer) th } } + /** + * Wraps an {@link InputStream} such that it's {@code close} method becomes a noop + * + * @param stream {@code InputStream} to wrap + * @return wrapped {@code InputStream} + */ + public static InputStream noCloseStream(InputStream stream) { + return new FilterInputStream(stream) { + @Override + public void close() { + // noop + } + }; + } + /** * Wraps the given {@link BytesStream} in a {@link StreamOutput} that simply flushes when * close is called. diff --git a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java index f51dfb62e0593..4548448719a68 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java @@ -196,6 +196,7 @@ public void apply(Settings value, Settings current, Settings previous) { IndicesQueryCache.INDICES_CACHE_QUERY_SIZE_SETTING, IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING, IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING, + IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING, MappingUpdatedAction.INDICES_MAPPING_DYNAMIC_TIMEOUT_SETTING, MetaData.SETTING_READ_ONLY_SETTING, MetaData.SETTING_READ_ONLY_ALLOW_DELETE_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexModule.java b/server/src/main/java/org/elasticsearch/index/IndexModule.java index b42bb2c2cb384..f1ce91b212b43 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexModule.java +++ b/server/src/main/java/org/elasticsearch/index/IndexModule.java @@ -74,6 +74,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiFunction; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; @@ -394,7 +395,8 @@ public IndexService newIndexService( IndicesQueryCache indicesQueryCache, MapperRegistry mapperRegistry, IndicesFieldDataCache indicesFieldDataCache, - NamedWriteableRegistry namedWriteableRegistry) + NamedWriteableRegistry namedWriteableRegistry, + BooleanSupplier idFieldDataEnabled) throws IOException { final IndexEventListener eventListener = freeze(); Function> readerWrapperFactory = @@ -422,7 +424,7 @@ public IndexService newIndexService( new SimilarityService(indexSettings, scriptService, similarities), shardStoreDeleter, indexAnalyzers, engineFactory, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, client, queryCache, directoryFactory, eventListener, readerWrapperFactory, mapperRegistry, indicesFieldDataCache, searchOperationListeners, - indexOperationListeners, namedWriteableRegistry); + indexOperationListeners, namedWriteableRegistry, idFieldDataEnabled); success = true; return indexService; } finally { @@ -469,7 +471,7 @@ public MapperService newIndexMapperService(NamedXContentRegistry xContentRegistr ScriptService scriptService) throws IOException { return new MapperService(indexSettings, analysisRegistry.build(indexSettings), xContentRegistry, new SimilarityService(indexSettings, scriptService, similarities), mapperRegistry, - () -> { throw new UnsupportedOperationException("no index query shard context available"); }); + () -> { throw new UnsupportedOperationException("no index query shard context available"); }, () -> false); } /** diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 7c29e342cb1de..47c3bfb35fee7 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -92,6 +92,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.LongSupplier; @@ -161,7 +162,8 @@ public IndexService( IndicesFieldDataCache indicesFieldDataCache, List searchOperationListeners, List indexingOperationListeners, - NamedWriteableRegistry namedWriteableRegistry) { + NamedWriteableRegistry namedWriteableRegistry, + BooleanSupplier idFieldDataEnabled) { super(indexSettings); this.indexSettings = indexSettings; this.xContentRegistry = xContentRegistry; @@ -172,7 +174,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null)); + () -> newQueryShardContext(0, null, System::currentTimeMillis, null), idFieldDataEnabled); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index f84079fc3f4d2..f319db284b2d2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -160,7 +160,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext if (nullValue != null) { boolean ignoreZValue = builder.ignoreZValue == null ? Defaults.IGNORE_Z_VALUE.value() : builder.ignoreZValue; - boolean ignoreMalformed = builder.ignoreMalformed == null ? Defaults.IGNORE_MALFORMED.value() : builder.ignoreZValue; + boolean ignoreMalformed = builder.ignoreMalformed == null ? Defaults.IGNORE_MALFORMED.value() : builder.ignoreMalformed; GeoPoint point = GeoUtils.parseGeoPoint(nullValue, ignoreZValue); if (ignoreMalformed == false) { if (point.lat() > 90.0 || point.lat() < -90.0) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java index e0a2cd7ee428b..521521915caf1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdFieldMapper.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.apache.logging.log4j.LogManager; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; @@ -28,6 +29,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.Index; @@ -41,6 +43,7 @@ import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.MultiValueMode; @@ -55,6 +58,11 @@ * queries. */ public class IdFieldMapper extends MetadataFieldMapper { + private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(IdFieldMapper.class)); + static final String ID_FIELD_DATA_DEPRECATION_MESSAGE = + "Loading the fielddata on the _id field is deprecated and will be removed in future versions. " + + "If you require sorting or aggregating on this field you should also include the id in the " + + "body of your documents, and map this field as a keyword field that has [doc_values] enabled"; public static final String NAME = "_id"; @@ -158,6 +166,12 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + if (mapperService.isIdFieldDataEnabled() == false) { + throw new IllegalArgumentException("Fielddata access on the _id field is disallowed, " + + "you can re-enable it by updating the dynamic cluster setting: " + + IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey()); + } + deprecationLogger.deprecatedAndMaybeLog("id_field_data", ID_FIELD_DATA_DEPRECATION_MESSAGE); final IndexFieldData fieldData = fieldDataBuilder.build(indexSettings, fieldType, cache, breakerService, mapperService); return new IndexFieldData() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 2a57cacf7ff45..4b29856ee94ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -67,6 +67,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Supplier; import java.util.stream.Stream; @@ -142,9 +143,11 @@ public enum MergeReason { final MapperRegistry mapperRegistry; + private final BooleanSupplier idFieldDataEnabled; + public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, NamedXContentRegistry xContentRegistry, SimilarityService similarityService, MapperRegistry mapperRegistry, - Supplier queryShardContextSupplier) { + Supplier queryShardContextSupplier, BooleanSupplier idFieldDataEnabled) { super(indexSettings); this.indexAnalyzers = indexAnalyzers; this.fieldTypes = new FieldTypeLookup(); @@ -154,12 +157,12 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer()); this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer()); this.mapperRegistry = mapperRegistry; + this.idFieldDataEnabled = idFieldDataEnabled; if (INDEX_MAPPER_DYNAMIC_SETTING.exists(indexSettings.getSettings()) && indexSettings.getIndexVersionCreated().onOrAfter(Version.V_7_0_0)) { throw new IllegalArgumentException("Setting " + INDEX_MAPPER_DYNAMIC_SETTING.getKey() + " was removed after version 6.0.0"); } - } public boolean hasNested() { @@ -663,6 +666,13 @@ public Analyzer searchQuoteAnalyzer() { return this.searchQuoteAnalyzer; } + /** + * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. + */ + public boolean isIdFieldDataEnabled() { + return idFieldDataEnabled.getAsBoolean(); + } + @Override public void close() throws IOException { indexAnalyzers.close(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 143e047e82147..46ef8a732a697 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -511,7 +511,7 @@ public String toString() { } } - public static final class TextFieldType extends StringFieldType { + public static class TextFieldType extends StringFieldType { private boolean fielddata; private double fielddataMinFrequency; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index b72da768a70dd..4810f9f00e8e5 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -92,6 +92,7 @@ import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.flush.FlushStats; import org.elasticsearch.index.get.GetStats; +import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.merge.MergeStats; import org.elasticsearch.index.query.QueryBuilder; @@ -168,6 +169,9 @@ public class IndicesService extends AbstractLifecycleComponent public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout"; public static final Setting INDICES_CACHE_CLEAN_INTERVAL_SETTING = Setting.positiveTimeSetting("indices.cache.cleanup_interval", TimeValue.timeValueMinutes(1), Property.NodeScope); + public static final Setting INDICES_ID_FIELD_DATA_ENABLED_SETTING = + Setting.boolSetting("indices.id_field_data.enabled", false, Property.Dynamic, Property.NodeScope); + /** * The node's settings. @@ -203,6 +207,7 @@ public class IndicesService extends AbstractLifecycleComponent private final Map directoryFactories; final AbstractRefCounted indicesRefCount; // pkg-private for testing private final CountDownLatch closeLatch = new CountDownLatch(1); + private volatile boolean idFieldDataEnabled; @Override protected void doStart() { @@ -238,6 +243,8 @@ public IndicesService(Settings settings, PluginsService pluginsService, NodeEnvi this.scriptService = scriptService; this.clusterService = clusterService; this.client = client; + this.idFieldDataEnabled = INDICES_ID_FIELD_DATA_ENABLED_SETTING.get(clusterService.getSettings()); + clusterService.getClusterSettings().addSettingsUpdateConsumer(INDICES_ID_FIELD_DATA_ENABLED_SETTING, this::setIdFieldDataEnabled); this.indicesFieldDataCache = new IndicesFieldDataCache(settings, new IndexFieldDataCache.Listener() { @Override public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) { @@ -558,7 +565,8 @@ private synchronized IndexService createIndexService(IndexService.IndexCreationC indicesQueryCache, mapperRegistry, indicesFieldDataCache, - namedWriteableRegistry + namedWriteableRegistry, + this::isIdFieldDataEnabled ); } @@ -1450,6 +1458,17 @@ public boolean isMetaDataField(Version indexCreatedVersion, String field) { return mapperRegistry.isMetaDataField(indexCreatedVersion, field); } + /** + * Returns true if fielddata is enabled for the {@link IdFieldMapper} field, false otherwise. + */ + public boolean isIdFieldDataEnabled() { + return idFieldDataEnabled; + } + + private void setIdFieldDataEnabled(boolean value) { + this.idFieldDataEnabled = value; + } + /** * Checks to see if an operation can be performed without taking the cluster over the cluster-wide shard limit. Adds a deprecation * warning or returns an error message as appropriate diff --git a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java index 504795d8d39a3..9cc414c5a15d6 100644 --- a/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/CompoundProcessor.java @@ -40,6 +40,7 @@ public class CompoundProcessor implements Processor { public static final String ON_FAILURE_MESSAGE_FIELD = "on_failure_message"; public static final String ON_FAILURE_PROCESSOR_TYPE_FIELD = "on_failure_processor_type"; public static final String ON_FAILURE_PROCESSOR_TAG_FIELD = "on_failure_processor_tag"; + public static final String ON_FAILURE_PIPELINE_FIELD = "on_failure_pipeline"; private final boolean ignoreFailure; private final List processors; @@ -144,7 +145,7 @@ void innerExecute(int currentProcessor, IngestDocument ingestDocument, BiConsume innerExecute(currentProcessor + 1, ingestDocument, handler); } else { IngestProcessorException compoundProcessorException = - newCompoundProcessorException(e, processor.getType(), processor.getTag()); + newCompoundProcessorException(e, processor, ingestDocument); if (onFailureProcessors.isEmpty()) { handler.accept(null, compoundProcessorException); } else { @@ -177,7 +178,7 @@ void executeOnFailureAsync(int currentOnFailureProcessor, IngestDocument ingestD onFailureProcessor.execute(ingestDocument, (result, e) -> { if (e != null) { removeFailureMetadata(ingestDocument); - handler.accept(null, newCompoundProcessorException(e, onFailureProcessor.getType(), onFailureProcessor.getTag())); + handler.accept(null, newCompoundProcessorException(e, onFailureProcessor, ingestDocument)); return; } if (result == null) { @@ -192,12 +193,17 @@ void executeOnFailureAsync(int currentOnFailureProcessor, IngestDocument ingestD private void putFailureMetadata(IngestDocument ingestDocument, ElasticsearchException cause) { List processorTypeHeader = cause.getHeader("processor_type"); List processorTagHeader = cause.getHeader("processor_tag"); + List processorOriginHeader = cause.getHeader("pipeline_origin"); String failedProcessorType = (processorTypeHeader != null) ? processorTypeHeader.get(0) : null; String failedProcessorTag = (processorTagHeader != null) ? processorTagHeader.get(0) : null; + String failedPipelineId = (processorOriginHeader != null) ? processorOriginHeader.get(0) : null; Map ingestMetadata = ingestDocument.getIngestMetadata(); ingestMetadata.put(ON_FAILURE_MESSAGE_FIELD, cause.getRootCause().getMessage()); ingestMetadata.put(ON_FAILURE_PROCESSOR_TYPE_FIELD, failedProcessorType); ingestMetadata.put(ON_FAILURE_PROCESSOR_TAG_FIELD, failedProcessorTag); + if (failedPipelineId != null) { + ingestMetadata.put(ON_FAILURE_PIPELINE_FIELD, failedPipelineId); + } } private void removeFailureMetadata(IngestDocument ingestDocument) { @@ -205,21 +211,28 @@ private void removeFailureMetadata(IngestDocument ingestDocument) { ingestMetadata.remove(ON_FAILURE_MESSAGE_FIELD); ingestMetadata.remove(ON_FAILURE_PROCESSOR_TYPE_FIELD); ingestMetadata.remove(ON_FAILURE_PROCESSOR_TAG_FIELD); + ingestMetadata.remove(ON_FAILURE_PIPELINE_FIELD); } - private IngestProcessorException newCompoundProcessorException(Exception e, String processorType, String processorTag) { + static IngestProcessorException newCompoundProcessorException(Exception e, Processor processor, IngestDocument document) { if (e instanceof IngestProcessorException && ((IngestProcessorException) e).getHeader("processor_type") != null) { return (IngestProcessorException) e; } IngestProcessorException exception = new IngestProcessorException(e); + String processorType = processor.getType(); if (processorType != null) { exception.addHeader("processor_type", processorType); } + String processorTag = processor.getTag(); if (processorTag != null) { exception.addHeader("processor_tag", processorTag); } + List pipelineStack = document.getPipelineStack(); + if (pipelineStack.size() > 1) { + exception.addHeader("pipeline_origin", pipelineStack); + } return exception; } diff --git a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java index c725157d8de0c..c2a5b09cd5352 100644 --- a/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java +++ b/server/src/main/java/org/elasticsearch/ingest/ConfigurationUtils.java @@ -348,6 +348,12 @@ public static List readProcessorConfigs(List> pro return processors; } + public static TemplateScript.Factory readTemplateProperty(String processorType, String processorTag, Map configuration, + String propertyName, ScriptService scriptService) { + String value = readStringProperty(processorType, processorTag, configuration, propertyName, null); + return compileTemplate(processorType, processorTag, propertyName, value, scriptService); + } + public static TemplateScript.Factory compileTemplate(String processorType, String processorTag, String propertyName, String propertyValue, ScriptService scriptService) { try { diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java index 6c8cacf14cdf5..aabb6890a7b29 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestDocument.java @@ -38,7 +38,7 @@ import java.util.Date; import java.util.EnumMap; import java.util.HashMap; -import java.util.IdentityHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -60,7 +60,7 @@ public final class IngestDocument { private final Map ingestMetadata; // Contains all pipelines that have been executed for this document - private final Set executedPipelines = Collections.newSetFromMap(new IdentityHashMap<>()); + private final Set executedPipelines = new LinkedHashSet<>(); public IngestDocument(String index, String id, String routing, Long version, VersionType versionType, Map source) { @@ -646,9 +646,9 @@ private static Object deepCopy(Object value) { * @param handler handles the result or failure */ public void executePipeline(Pipeline pipeline, BiConsumer handler) { - if (executedPipelines.add(pipeline)) { + if (executedPipelines.add(pipeline.getId())) { pipeline.execute(this, (result, e) -> { - executedPipelines.remove(pipeline); + executedPipelines.remove(pipeline.getId()); handler.accept(result, e); }); } else { @@ -656,6 +656,15 @@ public void executePipeline(Pipeline pipeline, BiConsumer getPipelineStack() { + List pipelineStack = new ArrayList<>(executedPipelines); + Collections.reverse(pipelineStack); + return pipelineStack; + } + @Override public boolean equals(Object obj) { if (obj == this) { return true; } diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index c8ef75d90a176..56b899f068b1c 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -460,7 +460,7 @@ public void addIngestClusterStateListener(Consumer listener) { } //package private for testing - static String getProcessorName(Processor processor){ + static String getProcessorName(Processor processor) { // conditionals are implemented as wrappers around the real processor, so get the real processor for the correct type for the name if(processor instanceof ConditionalProcessor){ processor = ((ConditionalProcessor) processor).getInnerProcessor(); @@ -469,7 +469,7 @@ static String getProcessorName(Processor processor){ sb.append(processor.getType()); if(processor instanceof PipelineProcessor){ - String pipelineName = ((PipelineProcessor) processor).getPipelineName(); + String pipelineName = ((PipelineProcessor) processor).getPipelineTemplate().newInstance(Map.of()).execute(); sb.append(":"); sb.append(pipelineName); } @@ -499,7 +499,7 @@ private void innerExecute(int slot, IndexRequest indexRequest, Pipeline pipeline VersionType versionType = indexRequest.versionType(); Map sourceAsMap = indexRequest.sourceAsMap(); IngestDocument ingestDocument = new IngestDocument(index, id, routing, version, versionType, sourceAsMap); - pipeline.execute(ingestDocument, (result, e) -> { + ingestDocument.executePipeline(pipeline, (result, e) -> { long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeInNanos); totalMetrics.postIngest(ingestTimeInMillis); if (e != null) { diff --git a/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java b/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java index f5e37a1c1235e..be02fe24752c1 100644 --- a/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/PipelineProcessor.java @@ -19,6 +19,8 @@ package org.elasticsearch.ingest; +import org.elasticsearch.script.TemplateScript; + import java.util.Map; import java.util.function.BiConsumer; @@ -26,18 +28,18 @@ public class PipelineProcessor extends AbstractProcessor { public static final String TYPE = "pipeline"; - private final String pipelineName; - + private final TemplateScript.Factory pipelineTemplate; private final IngestService ingestService; - private PipelineProcessor(String tag, String pipelineName, IngestService ingestService) { + private PipelineProcessor(String tag, TemplateScript.Factory pipelineTemplate, IngestService ingestService) { super(tag); - this.pipelineName = pipelineName; + this.pipelineTemplate = pipelineTemplate; this.ingestService = ingestService; } @Override public void execute(IngestDocument ingestDocument, BiConsumer handler) { + String pipelineName = ingestDocument.renderTemplate(this.pipelineTemplate); Pipeline pipeline = ingestService.getPipeline(pipelineName); if (pipeline != null) { ingestDocument.executePipeline(pipeline, handler); @@ -52,7 +54,8 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception { throw new UnsupportedOperationException("this method should not get executed"); } - Pipeline getPipeline(){ + Pipeline getPipeline(IngestDocument ingestDocument) { + String pipelineName = ingestDocument.renderTemplate(this.pipelineTemplate); return ingestService.getPipeline(pipelineName); } @@ -61,8 +64,8 @@ public String getType() { return TYPE; } - String getPipelineName() { - return pipelineName; + TemplateScript.Factory getPipelineTemplate() { + return pipelineTemplate; } public static final class Factory implements Processor.Factory { @@ -76,9 +79,9 @@ public Factory(IngestService ingestService) { @Override public PipelineProcessor create(Map registry, String processorTag, Map config) throws Exception { - String pipeline = - ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "name"); - return new PipelineProcessor(processorTag, pipeline, ingestService); + TemplateScript.Factory pipelineTemplate = + ConfigurationUtils.readTemplateProperty(TYPE, processorTag, config, "name", ingestService.getScriptService()); + return new PipelineProcessor(processorTag, pipelineTemplate, ingestService); } } } diff --git a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java index edd236c8c4e76..4abaadb353c55 100644 --- a/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java +++ b/server/src/main/java/org/elasticsearch/ingest/TrackingResultProcessor.java @@ -45,10 +45,10 @@ public final class TrackingResultProcessor implements Processor { public void execute(IngestDocument ingestDocument, BiConsumer handler) { if (actualProcessor instanceof PipelineProcessor) { PipelineProcessor pipelineProcessor = ((PipelineProcessor) actualProcessor); - Pipeline pipeline = pipelineProcessor.getPipeline(); + Pipeline pipeline = pipelineProcessor.getPipeline(ingestDocument); //runtime check for cycles against a copy of the document. This is needed to properly handle conditionals around pipelines IngestDocument ingestDocumentCopy = new IngestDocument(ingestDocument); - ingestDocumentCopy.executePipeline(pipelineProcessor.getPipeline(), (result, e) -> { + ingestDocumentCopy.executePipeline(pipelineProcessor.getPipeline(ingestDocument), (result, e) -> { // do nothing, let the tracking processors throw the exception while recording the path up to the failure if (e instanceof ElasticsearchException) { ElasticsearchException elasticsearchException = (ElasticsearchException) e; diff --git a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java index 0fe2f25fa2d22..e2a2d1be4ef07 100644 --- a/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java +++ b/server/src/main/java/org/elasticsearch/repositories/blobstore/BlobStoreRepository.java @@ -427,7 +427,7 @@ private RepositoryData safeRepositoryData(long repositoryStateId, Map foundIndices, Map rootBlobs, RepositoryData repositoryData, boolean writeShardGens, - ActionListener listener) throws IOException { + ActionListener listener) { if (writeShardGens) { // First write the new shard state metadata (with the removed snapshot) and compute deletion targets @@ -442,14 +442,14 @@ private void doDeleteShardSnapshots(SnapshotId snapshotId, long repositoryStateI // written if all shard paths have been successfully updated. final StepListener writeUpdatedRepoDataStep = new StepListener<>(); writeShardMetaDataAndComputeDeletesStep.whenComplete(deleteResults -> { - final ShardGenerations.Builder builder = ShardGenerations.builder(); - for (ShardSnapshotMetaDeleteResult newGen : deleteResults) { - builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); - } - final RepositoryData updatedRepoData = repositoryData.removeSnapshot(snapshotId, builder.build()); - writeIndexGen(updatedRepoData, repositoryStateId, true); - writeUpdatedRepoDataStep.onResponse(updatedRepoData); - }, listener::onFailure); + final ShardGenerations.Builder builder = ShardGenerations.builder(); + for (ShardSnapshotMetaDeleteResult newGen : deleteResults) { + builder.put(newGen.indexId, newGen.shardId, newGen.newGeneration); + } + final RepositoryData updatedRepoData = repositoryData.removeSnapshot(snapshotId, builder.build()); + writeIndexGen(updatedRepoData, repositoryStateId, true, + ActionListener.wrap(v -> writeUpdatedRepoDataStep.onResponse(updatedRepoData), listener::onFailure)); + }, listener::onFailure); // Once we have updated the repository, run the clean-ups writeUpdatedRepoDataStep.whenComplete(updatedRepoData -> { // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion @@ -461,15 +461,17 @@ private void doDeleteShardSnapshots(SnapshotId snapshotId, long repositoryStateI } else { // Write the new repository data first (with the removed snapshot), using no shard generations final RepositoryData updatedRepoData = repositoryData.removeSnapshot(snapshotId, ShardGenerations.EMPTY); - writeIndexGen(updatedRepoData, repositoryStateId, false); - // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion - final ActionListener afterCleanupsListener = - new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(null)), 2); - asyncCleanupUnlinkedRootAndIndicesBlobs(foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); - final StepListener> writeMetaAndComputeDeletesStep = new StepListener<>(); - writeUpdatedShardMetaDataAndComputeDeletes(snapshotId, repositoryData, false, writeMetaAndComputeDeletesStep); - writeMetaAndComputeDeletesStep.whenComplete(deleteResults -> - asyncCleanupUnlinkedShardLevelBlobs(snapshotId, deleteResults, afterCleanupsListener), afterCleanupsListener::onFailure); + writeIndexGen(updatedRepoData, repositoryStateId, false, ActionListener.wrap(v -> { + // Run unreferenced blobs cleanup in parallel to shard-level snapshot deletion + final ActionListener afterCleanupsListener = + new GroupedActionListener<>(ActionListener.wrap(() -> listener.onResponse(null)), 2); + asyncCleanupUnlinkedRootAndIndicesBlobs(foundIndices, rootBlobs, updatedRepoData, afterCleanupsListener); + final StepListener> writeMetaAndComputeDeletesStep = new StepListener<>(); + writeUpdatedShardMetaDataAndComputeDeletes(snapshotId, repositoryData, false, writeMetaAndComputeDeletesStep); + writeMetaAndComputeDeletesStep.whenComplete(deleteResults -> + asyncCleanupUnlinkedShardLevelBlobs(snapshotId, deleteResults, afterCleanupsListener), + afterCleanupsListener::onFailure); + }, listener::onFailure)); } } @@ -650,8 +652,9 @@ public void cleanup(long repositoryStateId, boolean writeShardGens, ActionListen listener.onResponse(new RepositoryCleanupResult(DeleteResult.ZERO)); } else { // write new index-N blob to ensure concurrent operations will fail - writeIndexGen(repositoryData, repositoryStateId, writeShardGens); - cleanupStaleBlobs(foundIndices, rootBlobs, repositoryData, ActionListener.map(listener, RepositoryCleanupResult::new)); + writeIndexGen(repositoryData, repositoryStateId, writeShardGens, + ActionListener.wrap(v -> cleanupStaleBlobs(foundIndices, rootBlobs, repositoryData, + ActionListener.map(listener, RepositoryCleanupResult::new)), listener::onFailure)); } } catch (Exception e) { listener.onFailure(e); @@ -762,11 +765,12 @@ public void finalizeSnapshot(final SnapshotId snapshotId, getRepositoryData(ActionListener.wrap(existingRepositoryData -> { final RepositoryData updatedRepositoryData = existingRepositoryData.addSnapshot(snapshotId, snapshotInfo.state(), shardGenerations); - writeIndexGen(updatedRepositoryData, repositoryStateId, writeShardGens); - if (writeShardGens) { - cleanupOldShardGens(existingRepositoryData, updatedRepositoryData); - } - listener.onResponse(snapshotInfo); + writeIndexGen(updatedRepositoryData, repositoryStateId, writeShardGens, ActionListener.wrap(v -> { + if (writeShardGens) { + cleanupOldShardGens(existingRepositoryData, updatedRepositoryData); + } + listener.onResponse(snapshotInfo); + }, onUpdateFailure)); }, onUpdateFailure)); }, onUpdateFailure), 2 + indices.size()); final Executor executor = threadPool.executor(ThreadPool.Names.SNAPSHOT); @@ -995,50 +999,58 @@ public boolean isReadOnly() { return readOnly; } - protected void writeIndexGen(final RepositoryData repositoryData, final long expectedGen, - final boolean writeShardGens) throws IOException { - assert isReadOnly() == false; // can not write to a read only repository - final long currentGen = repositoryData.getGenId(); - if (currentGen != expectedGen) { - // the index file was updated by a concurrent operation, so we were operating on stale - // repository data - throw new RepositoryException(metadata.name(), "concurrent modification of the index-N file, expected current generation [" + - expectedGen + "], actual current generation [" + currentGen + - "] - possibly due to simultaneous snapshot deletion requests"); - } - final long newGen = currentGen + 1; - if (latestKnownRepoGen.get() >= newGen) { - throw new IllegalArgumentException( - "Tried writing generation [" + newGen + "] but repository is at least at generation [" + newGen + "] already"); - } - // write the index file - final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen); - logger.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob); - writeAtomic(indexBlob, - BytesReference.bytes(repositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), writeShardGens)), true); - final long latestKnownGen = latestKnownRepoGen.updateAndGet(known -> Math.max(known, newGen)); - if (newGen < latestKnownGen) { - // Don't mess up the index.latest blob - throw new IllegalStateException( - "Wrote generation [" + newGen + "] but latest known repo gen concurrently changed to [" + latestKnownGen + "]"); - } - // write the current generation to the index-latest file - final BytesReference genBytes; - try (BytesStreamOutput bStream = new BytesStreamOutput()) { - bStream.writeLong(newGen); - genBytes = bStream.bytes(); - } - logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen); - writeAtomic(INDEX_LATEST_BLOB, genBytes, false); - // delete the N-2 index file if it exists, keep the previous one around as a backup - if (newGen - 2 >= 0) { - final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2); - try { - blobContainer().deleteBlobIgnoringIfNotExists(oldSnapshotIndexFile); - } catch (IOException e) { - logger.warn("Failed to clean up old index blob [{}]", oldSnapshotIndexFile); + /** + * @param repositoryData RepositoryData to write + * @param expectedGen expected repository generation at the start of the operation + * @param writeShardGens whether to write {@link ShardGenerations} to the new {@link RepositoryData} blob + * @param listener completion listener + */ + protected void writeIndexGen(RepositoryData repositoryData, long expectedGen, boolean writeShardGens, ActionListener listener) { + ActionListener.completeWith(listener, () -> { + assert isReadOnly() == false; // can not write to a read only repository + final long currentGen = repositoryData.getGenId(); + if (currentGen != expectedGen) { + // the index file was updated by a concurrent operation, so we were operating on stale + // repository data + throw new RepositoryException(metadata.name(), + "concurrent modification of the index-N file, expected current generation [" + expectedGen + + "], actual current generation [" + currentGen + "] - possibly due to simultaneous snapshot deletion requests"); } - } + final long newGen = currentGen + 1; + if (latestKnownRepoGen.get() >= newGen) { + throw new IllegalArgumentException( + "Tried writing generation [" + newGen + "] but repository is at least at generation [" + newGen + "] already"); + } + // write the index file + final String indexBlob = INDEX_FILE_PREFIX + Long.toString(newGen); + logger.debug("Repository [{}] writing new index generational blob [{}]", metadata.name(), indexBlob); + writeAtomic(indexBlob, + BytesReference.bytes(repositoryData.snapshotsToXContent(XContentFactory.jsonBuilder(), writeShardGens)), true); + final long latestKnownGen = latestKnownRepoGen.updateAndGet(known -> Math.max(known, newGen)); + if (newGen < latestKnownGen) { + // Don't mess up the index.latest blob + throw new IllegalStateException( + "Wrote generation [" + newGen + "] but latest known repo gen concurrently changed to [" + latestKnownGen + "]"); + } + // write the current generation to the index-latest file + final BytesReference genBytes; + try (BytesStreamOutput bStream = new BytesStreamOutput()) { + bStream.writeLong(newGen); + genBytes = bStream.bytes(); + } + logger.debug("Repository [{}] updating index.latest with generation [{}]", metadata.name(), newGen); + writeAtomic(INDEX_LATEST_BLOB, genBytes, false); + // delete the N-2 index file if it exists, keep the previous one around as a backup + if (newGen - 2 >= 0) { + final String oldSnapshotIndexFile = INDEX_FILE_PREFIX + Long.toString(newGen - 2); + try { + blobContainer().deleteBlobIgnoringIfNotExists(oldSnapshotIndexFile); + } catch (IOException e) { + logger.warn("Failed to clean up old index blob [{}]", oldSnapshotIndexFile); + } + } + return null; + }); } /** @@ -1432,7 +1444,7 @@ public void verify(String seed, DiscoveryNode localNode) { public String toString() { return "BlobStoreRepository[" + "[" + metadata.name() + - "], [" + blobStore() + ']' + + "], [" + blobStore.get() + ']' + ']'; } diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java index 2f9cd698cf3fb..d4a5a0cb58140 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestGetFieldMappingAction.java @@ -66,12 +66,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC client.admin().indices().getFieldMappings(getMappingsRequest, new RestBuilderListener<>(channel) { @Override public RestResponse buildResponse(GetFieldMappingsResponse response, XContentBuilder builder) throws Exception { - Map>> mappingsByIndex = response.mappings(); - - boolean isPossibleSingleFieldRequest = indices.length == 1 && fields.length == 1; - if (isPossibleSingleFieldRequest && isFieldMappingMissingField(mappingsByIndex)) { - return new BytesRestResponse(OK, builder.startObject().endObject()); - } + Map> mappingsByIndex = response.mappings(); RestStatus status = OK; if (mappingsByIndex.isEmpty() && fields.length > 0) { @@ -83,24 +78,4 @@ public RestResponse buildResponse(GetFieldMappingsResponse response, XContentBui }); } - /** - * Helper method to find out if the only included fieldmapping metadata is typed NULL, which means - * that type and index exist, but the field did not - */ - private boolean isFieldMappingMissingField(Map>> mappingsByIndex) { - if (mappingsByIndex.size() != 1) { - return false; - } - - for (Map> value : mappingsByIndex.values()) { - for (Map fieldValue : value.values()) { - for (Map.Entry fieldMappingMetaDataEntry : fieldValue.entrySet()) { - if (fieldMappingMetaDataEntry.getValue().isNull()) { - return true; - } - } - } - } - return false; - } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregator.java index 843e380e425ea..9dee689831ac9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/AvgAggregator.java @@ -73,6 +73,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, } final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx); + final CompensatedSum kahanSummation = new CompensatedSum(0, 0); + return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -87,7 +89,8 @@ public void collect(int doc, long bucket) throws IOException { // accurate than naive summation. double sum = sums.get(bucket); double compensation = compensations.get(bucket); - CompensatedSum kahanSummation = new CompensatedSum(sum, compensation); + + kahanSummation.reset(sum, compensation); for (int i = 0; i < valueCount; i++) { double value = values.nextValue(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CompensatedSum.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CompensatedSum.java index 965ac665159a0..85ae940681637 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CompensatedSum.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/CompensatedSum.java @@ -68,6 +68,14 @@ public CompensatedSum add(double value) { return add(value, NO_CORRECTION); } + /** + * Resets the internal state to use the new value and compensation delta + */ + public void reset(double value, double delta) { + this.value = value; + this.delta = delta; + } + /** * Increments the Kahan sum by adding two sums, and updating the correction term for reducing numeric errors. */ diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregator.java index c4dcfebf5e1be..f6b9420997fc0 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/ExtendedStatsAggregator.java @@ -90,6 +90,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, } final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx); + final CompensatedSum compensatedSum = new CompensatedSum(0, 0); + final CompensatedSum compensatedSumOfSqr = new CompensatedSum(0, 0); return new LeafBucketCollectorBase(sub, values) { @Override @@ -117,11 +119,11 @@ public void collect(int doc, long bucket) throws IOException { // which is more accurate than naive summation. double sum = sums.get(bucket); double compensation = compensations.get(bucket); - CompensatedSum compensatedSum = new CompensatedSum(sum, compensation); + compensatedSum.reset(sum, compensation); double sumOfSqr = sumOfSqrs.get(bucket); double compensationOfSqr = compensationOfSqrs.get(bucket); - CompensatedSum compensatedSumOfSqr = new CompensatedSum(sumOfSqr, compensationOfSqr); + compensatedSumOfSqr.reset(sumOfSqr, compensationOfSqr); for (int i = 0; i < valuesCount; i++) { double value = values.nextValue(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java index c7bcb4dc340a1..8675294d0fa87 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java @@ -68,6 +68,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol } final BigArrays bigArrays = context.bigArrays(); final MultiGeoValues values = valuesSource.geoValues(ctx); + final CompensatedSum compensatedSumLat = new CompensatedSum(0, 0); + final CompensatedSum compensatedSumLon = new CompensatedSum(0, 0); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -88,8 +90,8 @@ public void collect(int doc, long bucket) throws IOException { double sumLon = lonSum.get(bucket); double compensationLon = lonCompensations.get(bucket); - CompensatedSum compensatedSumLat = new CompensatedSum(sumLat, compensationLat); - CompensatedSum compensatedSumLon = new CompensatedSum(sumLon, compensationLon); + compensatedSumLat.reset(sumLat, compensationLat); + compensatedSumLon.reset(sumLon, compensationLon); // update the sum // diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java index 7799f498dd491..7ae5b016f75c3 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/StatsAggregator.java @@ -81,6 +81,8 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, } final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx); + final CompensatedSum kahanSummation = new CompensatedSum(0, 0); + return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -105,7 +107,7 @@ public void collect(int doc, long bucket) throws IOException { // accurate than naive summation. double sum = sums.get(bucket); double compensation = compensations.get(bucket); - CompensatedSum kahanSummation = new CompensatedSum(sum, compensation); + kahanSummation.reset(sum, compensation); for (int i = 0; i < valuesCount; i++) { double value = values.nextValue(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregator.java index cc440fd7d0554..ebb0e36dbf5db 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/SumAggregator.java @@ -69,6 +69,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, } final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues values = valuesSource.doubleValues(ctx); + final CompensatedSum kahanSummation = new CompensatedSum(0, 0); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -81,7 +82,7 @@ public void collect(int doc, long bucket) throws IOException { // accurate than naive summation. double sum = sums.get(bucket); double compensation = compensations.get(bucket); - CompensatedSum kahanSummation = new CompensatedSum(sum, compensation); + kahanSummation.reset(sum, compensation); for (int i = 0; i < valuesCount; i++) { double value = values.nextValue(); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregator.java index 11b4a5df951dd..ab5d1669e036f 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/WeightedAvgAggregator.java @@ -46,8 +46,8 @@ class WeightedAvgAggregator extends NumericMetricsAggregator.SingleValue { private final MultiValuesSource.NumericMultiValuesSource valuesSources; private DoubleArray weights; - private DoubleArray sums; - private DoubleArray sumCompensations; + private DoubleArray valueSums; + private DoubleArray valueCompensations; private DoubleArray weightCompensations; private DocValueFormat format; @@ -60,8 +60,8 @@ class WeightedAvgAggregator extends NumericMetricsAggregator.SingleValue { if (valuesSources != null) { final BigArrays bigArrays = context.bigArrays(); weights = bigArrays.newDoubleArray(1, true); - sums = bigArrays.newDoubleArray(1, true); - sumCompensations = bigArrays.newDoubleArray(1, true); + valueSums = bigArrays.newDoubleArray(1, true); + valueCompensations = bigArrays.newDoubleArray(1, true); weightCompensations = bigArrays.newDoubleArray(1, true); } } @@ -80,13 +80,15 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final BigArrays bigArrays = context.bigArrays(); final SortedNumericDoubleValues docValues = valuesSources.getField(VALUE_FIELD.getPreferredName(), ctx); final SortedNumericDoubleValues docWeights = valuesSources.getField(WEIGHT_FIELD.getPreferredName(), ctx); + final CompensatedSum compensatedValueSum = new CompensatedSum(0, 0); + final CompensatedSum compensatedWeightSum = new CompensatedSum(0, 0); return new LeafBucketCollectorBase(sub, docValues) { @Override public void collect(int doc, long bucket) throws IOException { weights = bigArrays.grow(weights, bucket + 1); - sums = bigArrays.grow(sums, bucket + 1); - sumCompensations = bigArrays.grow(sumCompensations, bucket + 1); + valueSums = bigArrays.grow(valueSums, bucket + 1); + valueCompensations = bigArrays.grow(valueCompensations, bucket + 1); weightCompensations = bigArrays.grow(weightCompensations, bucket + 1); if (docValues.advanceExact(doc) && docWeights.advanceExact(doc)) { @@ -102,42 +104,43 @@ public void collect(int doc, long bucket) throws IOException { final int numValues = docValues.docValueCount(); assert numValues > 0; + double valueSum = valueSums.get(bucket); + double valueCompensation = valueCompensations.get(bucket); + compensatedValueSum.reset(valueSum, valueCompensation); + + double weightSum = weights.get(bucket); + double weightCompensation = weightCompensations.get(bucket); + compensatedWeightSum.reset(weightSum, weightCompensation); + for (int i = 0; i < numValues; i++) { - kahanSum(docValues.nextValue() * weight, sums, sumCompensations, bucket); - kahanSum(weight, weights, weightCompensations, bucket); + compensatedValueSum.add(docValues.nextValue() * weight); + compensatedWeightSum.add(weight); } + + valueSums.set(bucket, compensatedValueSum.value()); + valueCompensations.set(bucket, compensatedValueSum.delta()); + weights.set(bucket, compensatedWeightSum.value()); + weightCompensations.set(bucket, compensatedWeightSum.delta()); } } }; } - private static void kahanSum(double value, DoubleArray values, DoubleArray compensations, long bucket) { - // Compute the sum of double values with Kahan summation algorithm which is more - // accurate than naive summation. - double sum = values.get(bucket); - double compensation = compensations.get(bucket); - - CompensatedSum kahanSummation = new CompensatedSum(sum, compensation) - .add(value); - - values.set(bucket, kahanSummation.value()); - compensations.set(bucket, kahanSummation.delta()); - } @Override public double metric(long owningBucketOrd) { - if (valuesSources == null || owningBucketOrd >= sums.size()) { + if (valuesSources == null || owningBucketOrd >= valueSums.size()) { return Double.NaN; } - return sums.get(owningBucketOrd) / weights.get(owningBucketOrd); + return valueSums.get(owningBucketOrd) / weights.get(owningBucketOrd); } @Override public InternalAggregation buildAggregation(long bucket) { - if (valuesSources == null || bucket >= sums.size()) { + if (valuesSources == null || bucket >= valueSums.size()) { return buildEmptyAggregation(); } - return new InternalWeightedAvg(name, sums.get(bucket), weights.get(bucket), format, pipelineAggregators(), metaData()); + return new InternalWeightedAvg(name, valueSums.get(bucket), weights.get(bucket), format, pipelineAggregators(), metaData()); } @Override @@ -147,7 +150,7 @@ public InternalAggregation buildEmptyAggregation() { @Override public void doClose() { - Releasables.close(weights, sums, sumCompensations, weightCompensations); + Releasables.close(weights, valueSums, valueCompensations, weightCompensations); } } diff --git a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 4bebdb0123672..85a0010fd58f9 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -27,6 +27,7 @@ import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.CollectionTerminatedException; import org.apache.lucene.search.Collector; +import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.ConjunctionDISI; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; @@ -35,9 +36,12 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryCache; import org.apache.lucene.search.QueryCachingPolicy; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.BitSet; @@ -45,14 +49,18 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.CombinedBitSet; import org.apache.lucene.util.SparseFixedBitSet; +import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.dfs.AggregatedDfs; import org.elasticsearch.search.profile.Timer; import org.elasticsearch.search.profile.query.ProfileWeight; import org.elasticsearch.search.profile.query.QueryProfileBreakdown; import org.elasticsearch.search.profile.query.QueryProfiler; import org.elasticsearch.search.profile.query.QueryTimingType; +import org.elasticsearch.search.query.QuerySearchResult; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Set; @@ -131,12 +139,86 @@ public Weight createWeight(Query query, ScoreMode scoreMode, float boost) throws } } + private void checkCancelled() { + if (checkCancelled != null) { + checkCancelled.run(); + } + } + + public void search(List leaves, Weight weight, CollectorManager manager, + QuerySearchResult result, DocValueFormat[] formats, TotalHits totalHits) throws IOException { + final List collectors = new ArrayList<>(leaves.size()); + for (LeafReaderContext ctx : leaves) { + final Collector collector = manager.newCollector(); + searchLeaf(ctx, weight, collector); + collectors.add(collector); + } + TopFieldDocs mergedTopDocs = (TopFieldDocs) manager.reduce(collectors); + // Lucene sets shards indexes during merging of topDocs from different collectors + // We need to reset shard index; ES will set shard index later during reduce stage + for (ScoreDoc scoreDoc : mergedTopDocs.scoreDocs) { + scoreDoc.shardIndex = -1; + } + if (totalHits != null) { // we have already precalculated totalHits for the whole index + mergedTopDocs = new TopFieldDocs(totalHits, mergedTopDocs.scoreDocs, mergedTopDocs.fields); + } + result.topDocs(new TopDocsAndMaxScore(mergedTopDocs, Float.NaN), formats); + } + @Override protected void search(List leaves, Weight weight, Collector collector) throws IOException { - final Weight cancellableWeight; - if (checkCancelled != null) { - cancellableWeight = new Weight(weight.getQuery()) { + for (LeafReaderContext ctx : leaves) { // search each subreader + searchLeaf(ctx, weight, collector); + } + } + + /** + * Lower-level search API. + * + * {@link LeafCollector#collect(int)} is called for every matching document in + * the provided ctx. + */ + private void searchLeaf(LeafReaderContext ctx, Weight weight, Collector collector) throws IOException { + checkCancelled(); + weight = wrapWeight(weight); + final LeafCollector leafCollector; + try { + leafCollector = collector.getLeafCollector(ctx); + } catch (CollectionTerminatedException e) { + // there is no doc of interest in this reader context + // continue with the following leaf + return; + } + Bits liveDocs = ctx.reader().getLiveDocs(); + BitSet liveDocsBitSet = getSparseBitSetOrNull(liveDocs); + if (liveDocsBitSet == null) { + BulkScorer bulkScorer = weight.bulkScorer(ctx); + if (bulkScorer != null) { + try { + bulkScorer.score(leafCollector, liveDocs); + } catch (CollectionTerminatedException e) { + // collection was terminated prematurely + // continue with the following leaf + } + } + } else { + // if the role query result set is sparse then we should use the SparseFixedBitSet for advancing: + Scorer scorer = weight.scorer(ctx); + if (scorer != null) { + try { + intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, + checkCancelled == null ? () -> { } : checkCancelled); + } catch (CollectionTerminatedException e) { + // collection was terminated prematurely + // continue with the following leaf + } + } + } + } + private Weight wrapWeight(Weight weight) { + if (checkCancelled != null) { + return new Weight(weight.getQuery()) { @Override public void extractTerms(Set terms) { throw new UnsupportedOperationException(); @@ -168,48 +250,10 @@ public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { } }; } else { - cancellableWeight = weight; + return weight; } - searchInternal(leaves, cancellableWeight, collector); } - private void searchInternal(List leaves, Weight weight, Collector collector) throws IOException { - for (LeafReaderContext ctx : leaves) { // search each subreader - final LeafCollector leafCollector; - try { - leafCollector = collector.getLeafCollector(ctx); - } catch (CollectionTerminatedException e) { - // there is no doc of interest in this reader context - // continue with the following leaf - continue; - } - Bits liveDocs = ctx.reader().getLiveDocs(); - BitSet liveDocsBitSet = getSparseBitSetOrNull(liveDocs); - if (liveDocsBitSet == null) { - BulkScorer bulkScorer = weight.bulkScorer(ctx); - if (bulkScorer != null) { - try { - bulkScorer.score(leafCollector, liveDocs); - } catch (CollectionTerminatedException e) { - // collection was terminated prematurely - // continue with the following leaf - } - } - } else { - // if the role query result set is sparse then we should use the SparseFixedBitSet for advancing: - Scorer scorer = weight.scorer(ctx); - if (scorer != null) { - try { - intersectScorerAndBitSet(scorer, liveDocsBitSet, leafCollector, - checkCancelled == null ? () -> {} : checkCancelled); - } catch (CollectionTerminatedException e) { - // collection was terminated prematurely - // continue with the following leaf - } - } - } - } - } private static BitSet getSparseBitSetOrNull(Bits liveDocs) { if (liveDocs instanceof SparseFixedBitSet) { diff --git a/server/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java b/server/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java index 19d382dd8f380..99490e3009fd7 100644 --- a/server/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java +++ b/server/src/main/java/org/elasticsearch/search/profile/query/CollectorResult.java @@ -49,8 +49,6 @@ public class CollectorResult implements ToXContentObject, Writeable { public static final String REASON_SEARCH_POST_FILTER = "search_post_filter"; public static final String REASON_SEARCH_MIN_SCORE = "search_min_score"; public static final String REASON_SEARCH_MULTI = "search_multi"; - public static final String REASON_SEARCH_TIMEOUT = "search_timeout"; - public static final String REASON_SEARCH_CANCELLED = "search_cancelled"; public static final String REASON_AGGREGATION = "aggregation"; public static final String REASON_AGGREGATION_GLOBAL = "aggregation_global"; diff --git a/server/src/main/java/org/elasticsearch/search/query/CancellableCollector.java b/server/src/main/java/org/elasticsearch/search/query/CancellableCollector.java deleted file mode 100644 index 504a7f3d13da5..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/query/CancellableCollector.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.query; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.FilterCollector; -import org.apache.lucene.search.LeafCollector; -import org.elasticsearch.tasks.TaskCancelledException; - -import java.io.IOException; -import java.util.function.BooleanSupplier; - -/** - * Collector that checks if the task it is executed under is cancelled. - */ -public class CancellableCollector extends FilterCollector { - private final BooleanSupplier cancelled; - - /** - * Constructor - * @param cancelled supplier of the cancellation flag, the supplier will be called for each segment - * @param in wrapped collector - */ - public CancellableCollector(BooleanSupplier cancelled, Collector in) { - super(in); - this.cancelled = cancelled; - } - - @Override - public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - if (cancelled.getAsBoolean()) { - throw new TaskCancelledException("cancelled"); - } - return super.getLeafCollector(context); - } -} diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryCollectorContext.java b/server/src/main/java/org/elasticsearch/search/query/QueryCollectorContext.java index f0c94bd822edf..b63739df76bfe 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryCollectorContext.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryCollectorContext.java @@ -28,16 +28,13 @@ import org.elasticsearch.common.lucene.MinimumScoreCollector; import org.elasticsearch.common.lucene.search.FilteredCollector; import org.elasticsearch.search.profile.query.InternalProfileCollector; -import org.elasticsearch.tasks.TaskCancelledException; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.function.BooleanSupplier; -import static org.elasticsearch.search.profile.query.CollectorResult.REASON_SEARCH_CANCELLED; import static org.elasticsearch.search.profile.query.CollectorResult.REASON_SEARCH_MIN_SCORE; import static org.elasticsearch.search.profile.query.CollectorResult.REASON_SEARCH_MULTI; import static org.elasticsearch.search.profile.query.CollectorResult.REASON_SEARCH_POST_FILTER; @@ -150,18 +147,6 @@ protected InternalProfileCollector createWithProfiler(InternalProfileCollector i }; } - /** - * Creates a collector that throws {@link TaskCancelledException} if the search is cancelled - */ - static QueryCollectorContext createCancellableCollectorContext(BooleanSupplier cancelled) { - return new QueryCollectorContext(REASON_SEARCH_CANCELLED) { - @Override - Collector create(Collector in) throws IOException { - return new CancellableCollector(cancelled, in); - } - }; - } - /** * Creates collector limiting the collection to the first numHits documents */ diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 7f3a7a5b1b513..81905ed3345a9 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -21,26 +21,40 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; import org.apache.lucene.queries.MinDocQuery; import org.apache.lucene.queries.SearchAfterSortedDocQuery; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; +import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.search.TopFieldDocs; import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.Weight; import org.elasticsearch.action.search.SearchTask; +import org.elasticsearch.common.Booleans; +import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.TopDocsAndMaxScore; import org.elasticsearch.common.util.concurrent.QueueResizingEsThreadPoolExecutor; +import org.elasticsearch.index.IndexSortConfig; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchPhase; import org.elasticsearch.search.SearchService; @@ -57,16 +71,21 @@ import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.threadpool.ThreadPool; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.LinkedList; +import java.util.List; import java.util.concurrent.ExecutorService; -import java.util.function.Consumer; -import static org.elasticsearch.search.query.QueryCollectorContext.createCancellableCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createEarlyTerminationCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createFilteredCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createMinScoreCollectorContext; import static org.elasticsearch.search.query.QueryCollectorContext.createMultiCollectorContext; import static org.elasticsearch.search.query.TopDocsCollectorContext.createTopDocsCollectorContext; +import static org.elasticsearch.search.query.TopDocsCollectorContext.shortcutTotalHitCount; /** @@ -75,6 +94,8 @@ */ public class QueryPhase implements SearchPhase { private static final Logger LOGGER = LogManager.getLogger(QueryPhase.class); + // TODO: remove this property in 8.0 + public static final boolean SYS_PROP_REWRITE_SORT = Booleans.parseBoolean(System.getProperty("es.search.rewrite_sort", "true")); private final AggregationPhase aggregationPhase; private final SuggestPhase suggestPhase; @@ -97,7 +118,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep suggestPhase.execute(searchContext); searchContext.queryResult().topDocs(new TopDocsAndMaxScore( new TopDocs(new TotalHits(0, TotalHits.Relation.EQUAL_TO), Lucene.EMPTY_SCORE_DOCS), Float.NaN), - new DocValueFormat[0]); + new DocValueFormat[0]); return; } @@ -109,8 +130,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep // request, preProcess is called on the DFS phase phase, this is why we pre-process them // here to make sure it happens during the QUERY phase aggregationPhase.preProcess(searchContext); - final ContextIndexSearcher searcher = searchContext.searcher(); - boolean rescore = execute(searchContext, searchContext.searcher(), searcher::setCheckCancelled); + boolean rescore = executeInternal(searchContext); if (rescore) { // only if we do a regular search rescorePhase.execute(searchContext); @@ -120,7 +140,7 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep if (searchContext.getProfilers() != null) { ProfileShardResult shardResults = SearchProfileShardResults - .buildShardResults(searchContext.getProfilers()); + .buildShardResults(searchContext.getProfilers()); searchContext.queryResult().profileResults(shardResults); } } @@ -130,9 +150,9 @@ public void execute(SearchContext searchContext) throws QueryPhaseExecutionExcep * wire everything (mapperService, etc.) * @return whether the rescoring phase should be executed */ - static boolean execute(SearchContext searchContext, - final IndexSearcher searcher, - Consumer checkCancellationSetter) throws QueryPhaseExecutionException { + static boolean executeInternal(SearchContext searchContext) throws QueryPhaseExecutionException { + final ContextIndexSearcher searcher = searchContext.searcher(); + SortAndFormats sortAndFormatsForRewrittenNumericSort = null; final IndexReader reader = searcher.getIndexReader(); QuerySearchResult queryResult = searchContext.queryResult(); queryResult.searchTimedOut(false); @@ -204,6 +224,27 @@ static boolean execute(SearchContext searchContext, hasFilterCollector = true; } + CheckedConsumer, IOException> leafSorter = l -> {}; + // try to rewrite numeric or date sort to the optimized distanceFeatureQuery + if ((searchContext.sort() != null) && SYS_PROP_REWRITE_SORT) { + Query rewrittenQuery = tryRewriteLongSort(searchContext, searcher.getIndexReader(), query, hasFilterCollector); + if (rewrittenQuery != null) { + query = rewrittenQuery; + // modify sorts: add sort on _score as 1st sort, and move the sort on the original field as the 2nd sort + SortField[] oldSortFields = searchContext.sort().sort.getSort(); + DocValueFormat[] oldFormats = searchContext.sort().formats; + SortField[] newSortFields = new SortField[oldSortFields.length + 1]; + DocValueFormat[] newFormats = new DocValueFormat[oldSortFields.length + 1]; + newSortFields[0] = SortField.FIELD_SCORE; + newFormats[0] = DocValueFormat.RAW; + System.arraycopy(oldSortFields, 0, newSortFields, 1, oldSortFields.length); + System.arraycopy(oldFormats, 0, newFormats, 1, oldFormats.length); + sortAndFormatsForRewrittenNumericSort = searchContext.sort(); // stash SortAndFormats to restore it later + searchContext.sort(new SortAndFormats(new Sort(newSortFields), newFormats)); + leafSorter = createLeafSorter(oldSortFields[0]); + } + } + boolean timeoutSet = scrollContext == null && searchContext.timeout() != null && searchContext.timeout().equals(SearchService.NO_TIMEOUT) == false; @@ -243,53 +284,22 @@ static boolean execute(SearchContext searchContext, } else { checkCancelled = null; } + searcher.setCheckCancelled(checkCancelled); - checkCancellationSetter.accept(checkCancelled); - - // add cancellable - // this only performs segment-level cancellation, which is cheap and checked regardless of - // searchContext.lowLevelCancellation() - collectors.add(createCancellableCollectorContext(searchContext.getTask()::isCancelled)); - - final boolean doProfile = searchContext.getProfilers() != null; - // create the top docs collector last when the other collectors are known - final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, reader, hasFilterCollector); - // add the top docs collector, the first collector context in the chain - collectors.addFirst(topDocsFactory); - - final Collector queryCollector; - if (doProfile) { - InternalProfileCollector profileCollector = QueryCollectorContext.createQueryCollectorWithProfiler(collectors); - searchContext.getProfilers().getCurrentQueryProfiler().setCollector(profileCollector); - queryCollector = profileCollector; + boolean shouldRescore; + // if we are optimizing sort and there are no other collectors + if (sortAndFormatsForRewrittenNumericSort != null && collectors.size() == 0 && searchContext.getProfilers() == null) { + shouldRescore = searchWithCollectorManager(searchContext, searcher, query, leafSorter, timeoutSet); } else { - queryCollector = QueryCollectorContext.createQueryCollector(collectors); + shouldRescore = searchWithCollector(searchContext, searcher, query, collectors, hasFilterCollector, timeoutSet); } - try { - searcher.search(query, queryCollector); - } catch (EarlyTerminatingCollector.EarlyTerminationException e) { - queryResult.terminatedEarly(true); - } catch (TimeExceededException e) { - assert timeoutSet : "TimeExceededException thrown even though timeout wasn't set"; - - if (searchContext.request().allowPartialSearchResults() == false) { - // Can't rethrow TimeExceededException because not serializable - throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded"); - } - queryResult.searchTimedOut(true); - } finally { - searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION); - } - if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER - && queryResult.terminatedEarly() == null) { - queryResult.terminatedEarly(false); + // if we rewrote numeric long or date sort, restore fieldDocs based on the original sort + if (sortAndFormatsForRewrittenNumericSort != null) { + searchContext.sort(sortAndFormatsForRewrittenNumericSort); // restore SortAndFormats + restoreTopFieldDocs(queryResult, sortAndFormatsForRewrittenNumericSort); } - final QuerySearchResult result = searchContext.queryResult(); - for (QueryCollectorContext ctx : collectors) { - ctx.postProcess(result); - } ExecutorService executor = searchContext.indexShard().getThreadPool().executor(ThreadPool.Names.SEARCH); if (executor instanceof QueueResizingEsThreadPoolExecutor) { QueueResizingEsThreadPoolExecutor rExecutor = (QueueResizingEsThreadPoolExecutor) executor; @@ -298,14 +308,222 @@ static boolean execute(SearchContext searchContext, } if (searchContext.getProfilers() != null) { ProfileShardResult shardResults = SearchProfileShardResults.buildShardResults(searchContext.getProfilers()); - result.profileResults(shardResults); + queryResult.profileResults(shardResults); } - return topDocsFactory.shouldRescore(); + return shouldRescore; } catch (Exception e) { throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Failed to execute main query", e); } } + private static boolean searchWithCollector(SearchContext searchContext, ContextIndexSearcher searcher, Query query, + LinkedList collectors, boolean hasFilterCollector, boolean timeoutSet) throws IOException { + // create the top docs collector last when the other collectors are known + final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector); + // add the top docs collector, the first collector context in the chain + collectors.addFirst(topDocsFactory); + + final Collector queryCollector; + if (searchContext.getProfilers() != null) { + InternalProfileCollector profileCollector = QueryCollectorContext.createQueryCollectorWithProfiler(collectors); + searchContext.getProfilers().getCurrentQueryProfiler().setCollector(profileCollector); + queryCollector = profileCollector; + } else { + queryCollector = QueryCollectorContext.createQueryCollector(collectors); + } + QuerySearchResult queryResult = searchContext.queryResult(); + try { + searcher.search(query, queryCollector); + } catch (EarlyTerminatingCollector.EarlyTerminationException e) { + queryResult.terminatedEarly(true); + } catch (TimeExceededException e) { + assert timeoutSet : "TimeExceededException thrown even though timeout wasn't set"; + if (searchContext.request().allowPartialSearchResults() == false) { + // Can't rethrow TimeExceededException because not serializable + throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded"); + } + queryResult.searchTimedOut(true); + } finally { + searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION); + } + if (searchContext.terminateAfter() != SearchContext.DEFAULT_TERMINATE_AFTER && queryResult.terminatedEarly() == null) { + queryResult.terminatedEarly(false); + } + for (QueryCollectorContext ctx : collectors) { + ctx.postProcess(queryResult); + } + return topDocsFactory.shouldRescore(); + } + + + /* + * We use collectorManager during sort optimization, where + * we have already checked that there are no other collectors, no filters, + * no search after, no scroll, no collapse, no track scores. + * Absence of all other collectors and parameters allows us to use TopFieldCollector directly. + */ + private static boolean searchWithCollectorManager(SearchContext searchContext, ContextIndexSearcher searcher, Query query, + CheckedConsumer, IOException> leafSorter, boolean timeoutSet) throws IOException { + final IndexReader reader = searchContext.searcher().getIndexReader(); + final int numHits = Math.min(searchContext.from() + searchContext.size(), Math.max(1, reader.numDocs())); + final SortAndFormats sortAndFormats = searchContext.sort(); + + int totalHitsThreshold; + TotalHits totalHits; + if (searchContext.trackTotalHitsUpTo() == SearchContext.TRACK_TOTAL_HITS_DISABLED) { + totalHitsThreshold = 1; + totalHits = new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO); + } else { + int hitCount = shortcutTotalHitCount(reader, query); + if (hitCount == -1) { + totalHitsThreshold = searchContext.trackTotalHitsUpTo(); + totalHits = null; // will be computed via the collector + } else { + totalHitsThreshold = 1; + totalHits = new TotalHits(hitCount, TotalHits.Relation.EQUAL_TO); // don't compute hit counts via the collector + } + } + + CollectorManager sharedManager = TopFieldCollector.createSharedManager( + sortAndFormats.sort, numHits, null, totalHitsThreshold); + + List leaves = new ArrayList<>(searcher.getIndexReader().leaves()); + leafSorter.accept(leaves); + try { + Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.TOP_SCORES, 1f); + searcher.search(leaves, weight, sharedManager, searchContext.queryResult(), sortAndFormats.formats, totalHits); + } catch (TimeExceededException e) { + assert timeoutSet : "TimeExceededException thrown even though timeout wasn't set"; + if (searchContext.request().allowPartialSearchResults() == false) { + // Can't rethrow TimeExceededException because not serializable + throw new QueryPhaseExecutionException(searchContext.shardTarget(), "Time exceeded"); + } + searchContext.queryResult().searchTimedOut(true); + } finally { + searchContext.clearReleasables(SearchContext.Lifetime.COLLECTION); + } + return false; // no rescoring when sorting by field + } + + private static Query tryRewriteLongSort(SearchContext searchContext, IndexReader reader, + Query query, boolean hasFilterCollector) throws IOException { + if (searchContext.searchAfter() != null) return null; //TODO: handle sort optimization with search after + if (searchContext.scrollContext() != null) return null; + if (searchContext.collapse() != null) return null; + if (searchContext.trackScores()) return null; + if (searchContext.aggregations() != null) return null; + Sort sort = searchContext.sort().sort; + SortField sortField = sort.getSort()[0]; + if (SortField.Type.LONG.equals(IndexSortConfig.getSortFieldType(sortField)) == false) return null; + + // check if this is a field of type Long or Date, that is indexed and has doc values + String fieldName = sortField.getField(); + if (fieldName == null) return null; // happens when _score or _doc is the 1st sort field + if (searchContext.mapperService() == null) return null; // mapperService can be null in tests + final MappedFieldType fieldType = searchContext.mapperService().fullName(fieldName); + if (fieldType == null) return null; // for unmapped fields, default behaviour depending on "unmapped_type" flag + if ((fieldType.typeName().equals("long") == false) && (fieldType instanceof DateFieldType == false)) return null; + if (fieldType.indexOptions() == IndexOptions.NONE) return null; //TODO: change to pointDataDimensionCount() when implemented + if (fieldType.hasDocValues() == false) return null; + + + // check that all sorts are actual document fields or _doc + for (int i = 1; i < sort.getSort().length; i++) { + SortField sField = sort.getSort()[i]; + String sFieldName = sField.getField(); + if (sFieldName == null) { + if (SortField.FIELD_DOC.equals(sField) == false) return null; + } else { + //TODO: find out how to cover _script sort that don't use _score + if (searchContext.mapperService().fullName(sFieldName) == null) return null; // could be _script sort that uses _score + } + } + + // check that setting of missing values allows optimization + if (sortField.getMissingValue() == null) return null; + Long missingValue = (Long) sortField.getMissingValue(); + boolean missingValuesAccordingToSort = (sortField.getReverse() && (missingValue == Long.MIN_VALUE)) || + ((sortField.getReverse() == false) && (missingValue == Long.MAX_VALUE)); + if (missingValuesAccordingToSort == false) return null; + + int docCount = PointValues.getDocCount(reader, fieldName); + // is not worth to run optimization on small index + if (docCount <= 512) return null; + + // check for multiple values + if (PointValues.size(reader, fieldName) != docCount) return null; //TODO: handle multiple values + + // check if the optimization makes sense with the track_total_hits setting + if (searchContext.trackTotalHitsUpTo() == Integer.MAX_VALUE) { + // with filter, we can't pre-calculate hitsCount, we need to explicitly calculate them => optimization does't make sense + if (hasFilterCollector) return null; + // if we can't pre-calculate hitsCount based on the query type, optimization does't make sense + if (shortcutTotalHitCount(reader, query) == -1) return null; + } + + byte[] minValueBytes = PointValues.getMinPackedValue(reader, fieldName); + byte[] maxValueBytes = PointValues.getMaxPackedValue(reader, fieldName); + if ((maxValueBytes == null) || (minValueBytes == null)) return null; + long minValue = LongPoint.decodeDimension(minValueBytes, 0); + long maxValue = LongPoint.decodeDimension(maxValueBytes, 0); + + Query rewrittenQuery; + if (minValue == maxValue) { + rewrittenQuery = new DocValuesFieldExistsQuery(fieldName); + } else { + if (indexFieldHasDuplicateData(reader, fieldName)) return null; + long origin = (sortField.getReverse()) ? maxValue : minValue; + long pivotDistance = (maxValue - minValue) >>> 1; // division by 2 on the unsigned representation to avoid overflow + if (pivotDistance == 0) { // 0 if maxValue = (minValue + 1) + pivotDistance = 1; + } + rewrittenQuery = LongPoint.newDistanceFeatureQuery(sortField.getField(), 1, origin, pivotDistance); + } + rewrittenQuery = new BooleanQuery.Builder() + .add(query, BooleanClause.Occur.FILTER) // filter for original query + .add(rewrittenQuery, BooleanClause.Occur.SHOULD) //should for rewrittenQuery + .build(); + return rewrittenQuery; + } + + /** + * Creates a sorter of {@link LeafReaderContext} that orders leaves depending on the minimum + * value and the sort order of the provided sortField. + */ + static CheckedConsumer, IOException> createLeafSorter(SortField sortField) { + return leaves -> { + long[] sortValues = new long[leaves.size()]; + long missingValue = (long) sortField.getMissingValue(); + for (LeafReaderContext ctx : leaves) { + PointValues values = ctx.reader().getPointValues(sortField.getField()); + if (values == null) { + sortValues[ctx.ord] = missingValue; + } else { + byte[] sortValue = sortField.getReverse() ? values.getMaxPackedValue(): values.getMinPackedValue(); + sortValues[ctx.ord] = sortValue == null ? missingValue : LongPoint.decodeDimension(sortValue, 0); + } + } + Comparator comparator = Comparator.comparingLong(l -> sortValues[l.ord]); + if (sortField.getReverse()) { + comparator = comparator.reversed(); + } + Collections.sort(leaves, comparator); + }; + } + + /** + * Restore fieldsDocs to remove the first _score + */ + private static void restoreTopFieldDocs(QuerySearchResult result, SortAndFormats originalSortAndFormats) { + TopDocs topDocs = result.topDocs().topDocs; + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + FieldDoc fieldDoc = (FieldDoc) scoreDoc; + fieldDoc.fields = Arrays.copyOfRange(fieldDoc.fields, 1, fieldDoc.fields.length); + } + TopFieldDocs newTopDocs = new TopFieldDocs(topDocs.totalHits, topDocs.scoreDocs, originalSortAndFormats.sort.getSort()); + result.topDocs(new TopDocsAndMaxScore(newTopDocs, Float.NaN), originalSortAndFormats.formats); + } + /** * Returns true if the provided query returns docs in index order (internal doc ids). * @param query The query to execute @@ -341,5 +559,79 @@ private static boolean canEarlyTerminate(IndexReader reader, SortAndFormats sort return true; } + /** + * Returns true if more than 50% of data in the index have the same value + * The evaluation is approximation based on finding the median value and estimating its count + */ + static boolean indexFieldHasDuplicateData(IndexReader reader, String field) throws IOException { + long docsNoDupl = 0; // number of docs in segments with NO duplicate data that would benefit optimization + long docsDupl = 0; // number of docs in segments with duplicate data that would NOT benefit optimization + for (LeafReaderContext lrc : reader.leaves()) { + PointValues pointValues = lrc.reader().getPointValues(field); + if (pointValues == null) continue; + int docCount = pointValues.getDocCount(); + if (docCount <= 512) { // skipping small segments as estimateMedianCount doesn't work well on them + continue; + } + assert(pointValues.size() == docCount); // TODO: modify the code to handle multiple values + + int duplDocCount = docCount/2; // expected doc count of duplicate data + long minValue = LongPoint.decodeDimension(pointValues.getMinPackedValue(), 0); + long maxValue = LongPoint.decodeDimension(pointValues.getMaxPackedValue(), 0); + boolean hasDuplicateData = true; + while ((minValue < maxValue) && hasDuplicateData) { + long midValue = Math.floorDiv(minValue, 2) + Math.floorDiv(maxValue, 2); // to avoid overflow first divide each value by 2 + long countLeft = estimatePointCount(pointValues, minValue, midValue); + long countRight = estimatePointCount(pointValues, midValue + 1, maxValue); + if ((countLeft >= countRight) && (countLeft > duplDocCount) ) { + maxValue = midValue; + } else if ((countRight > countLeft) && (countRight > duplDocCount)) { + minValue = midValue + 1; + } else { + hasDuplicateData = false; + } + } + if (hasDuplicateData) { + docsDupl += docCount; + } else { + docsNoDupl += docCount; + } + } + return (docsDupl > docsNoDupl); + } + + + private static long estimatePointCount(PointValues pointValues, long minValue, long maxValue) { + final byte[] minValueAsBytes = new byte[Long.BYTES]; + LongPoint.encodeDimension(minValue, minValueAsBytes, 0); + final byte[] maxValueAsBytes = new byte[Long.BYTES]; + LongPoint.encodeDimension(maxValue, maxValueAsBytes, 0); + + PointValues.IntersectVisitor visitor = new PointValues.IntersectVisitor() { + @Override + public void grow(int count) {} + + @Override + public void visit(int docID) {} + + @Override + public void visit(int docID, byte[] packedValue) {} + + @Override + public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) { + if (Arrays.compareUnsigned(minPackedValue, 0, Long.BYTES, maxValueAsBytes, 0, Long.BYTES) > 0 || + Arrays.compareUnsigned(maxPackedValue, 0, Long.BYTES, minValueAsBytes, 0, Long.BYTES) < 0) { + return PointValues.Relation.CELL_OUTSIDE_QUERY; + } + if (Arrays.compareUnsigned(minPackedValue, 0, Long.BYTES, minValueAsBytes, 0, Long.BYTES) < 0 || + Arrays.compareUnsigned(maxPackedValue, 0, Long.BYTES, maxValueAsBytes, 0, Long.BYTES) > 0) { + return PointValues.Relation.CELL_CROSSES_QUERY; + } + return PointValues.Relation.CELL_INSIDE_QUERY; + } + }; + return pointValues.estimatePointCount(visitor); + } + private static class TimeExceededException extends RuntimeException {} } diff --git a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java index 751c1cd8bfbe7..f912d1e99129a 100644 --- a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java +++ b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java @@ -414,8 +414,8 @@ static int shortcutTotalHitCount(IndexReader reader, Query query) throws IOExcep * @param hasFilterCollector True if the collector chain contains at least one collector that can filters document. */ static TopDocsCollectorContext createTopDocsCollectorContext(SearchContext searchContext, - IndexReader reader, boolean hasFilterCollector) throws IOException { + final IndexReader reader = searchContext.searcher().getIndexReader(); final Query query = searchContext.query(); // top collectors don't like a size of 0 final int totalNumDocs = Math.max(1, reader.numDocs()); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java index 2e356c06b6eee..dc85e3c4d974a 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/mapping/get/GetFieldMappingsResponseTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.indices.mapping.get; import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -34,22 +35,29 @@ public class GetFieldMappingsResponseTests extends AbstractWireSerializingTestCase { public void testManualSerialization() throws IOException { - Map>> mappings = new HashMap<>(); + Map> mappings = new HashMap<>(); FieldMappingMetaData fieldMappingMetaData = new FieldMappingMetaData("my field", new BytesArray("{}")); - mappings.put("index", Collections.singletonMap("type", Collections.singletonMap("field", fieldMappingMetaData))); + mappings.put("index", Collections.singletonMap("field", fieldMappingMetaData)); GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings); try (BytesStreamOutput out = new BytesStreamOutput()) { response.writeTo(out); try (StreamInput in = StreamInput.wrap(out.bytes().toBytesRef().bytes)) { GetFieldMappingsResponse serialized = new GetFieldMappingsResponse(in); - FieldMappingMetaData metaData = serialized.fieldMappings("index", "type", "field"); + FieldMappingMetaData metaData = serialized.fieldMappings("index", "field"); assertNotNull(metaData); assertEquals(new BytesArray("{}"), metaData.getSource()); } } } + public void testNullFieldMappingToXContent() { + Map> mappings = new HashMap<>(); + mappings.put("index", Collections.emptyMap()); + GetFieldMappingsResponse response = new GetFieldMappingsResponse(mappings); + assertEquals("{\"index\":{\"mappings\":{}}}", Strings.toString(response)); + } + @Override protected GetFieldMappingsResponse createTestInstance() { return new GetFieldMappingsResponse(randomMapping()); @@ -60,25 +68,20 @@ protected Writeable.Reader instanceReader() { return GetFieldMappingsResponse::new; } - private Map>> randomMapping() { - Map>> mappings = new HashMap<>(); + private Map> randomMapping() { + Map> mappings = new HashMap<>(); int indices = randomInt(10); for(int i = 0; i < indices; i++) { - final Map> doctypesMappings = new HashMap<>(); - int doctypes = randomInt(10); - for(int j = 0; j < doctypes; j++) { - Map fieldMappings = new HashMap<>(); - int fields = randomInt(10); - for(int k = 0; k < fields; k++) { - final String mapping = randomBoolean() ? "{\"type\":\"string\"}" : "{\"type\":\"keyword\"}"; - FieldMappingMetaData metaData = - new FieldMappingMetaData("my field", new BytesArray(mapping)); - fieldMappings.put("field" + k, metaData); - } - doctypesMappings.put("doctype" + j, fieldMappings); + Map fieldMappings = new HashMap<>(); + int fields = randomInt(10); + for (int k = 0; k < fields; k++) { + final String mapping = randomBoolean() ? "{\"type\":\"string\"}" : "{\"type\":\"keyword\"}"; + FieldMappingMetaData metaData = + new FieldMappingMetaData("my field", new BytesArray(mapping)); + fieldMappings.put("field" + k, metaData); } - mappings.put("index" + i, doctypesMappings); + mappings.put("index" + i, fieldMappings); } return mappings; } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java index 395fe37f43167..a9aca8a28179a 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkWithUpdatesIT.java @@ -199,6 +199,43 @@ public void testBulkUpdateSimple() throws Exception { assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(4L)); } + public void testBulkUpdateWithScriptedUpsertAndDynamicMappingUpdate() throws Exception { + assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); + ensureGreen(); + + final Script script = new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "ctx._source.field += 1", Collections.emptyMap()); + + BulkResponse bulkResponse = client().prepareBulk() + .add(client().prepareUpdate().setIndex(indexOrAlias()).setId("1") + .setScript(script).setScriptedUpsert(true).setUpsert("field", 1)) + .add(client().prepareUpdate().setIndex(indexOrAlias()).setId("2") + .setScript(script).setScriptedUpsert(true).setUpsert("field", 1)) + .get(); + + logger.info(bulkResponse.buildFailureMessage()); + + assertThat(bulkResponse.hasFailures(), equalTo(false)); + assertThat(bulkResponse.getItems().length, equalTo(2)); + for (BulkItemResponse bulkItemResponse : bulkResponse) { + assertThat(bulkItemResponse.getIndex(), equalTo("test")); + } + assertThat(bulkResponse.getItems()[0].getResponse().getId(), equalTo("1")); + assertThat(bulkResponse.getItems()[0].getResponse().getVersion(), equalTo(1L)); + assertThat(bulkResponse.getItems()[1].getResponse().getId(), equalTo("2")); + assertThat(bulkResponse.getItems()[1].getResponse().getVersion(), equalTo(1L)); + + GetResponse getResponse = client().prepareGet().setIndex("test").setId("1").execute() + .actionGet(); + assertThat(getResponse.isExists(), equalTo(true)); + assertThat(getResponse.getVersion(), equalTo(1L)); + assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(2L)); + + getResponse = client().prepareGet().setIndex("test").setId("2").execute().actionGet(); + assertThat(getResponse.isExists(), equalTo(true)); + assertThat(getResponse.getVersion(), equalTo(1L)); + assertThat(((Number) getResponse.getSource().get("field")).longValue(), equalTo(2L)); + } + public void testBulkWithCAS() throws Exception { createIndex("test", Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build()); ensureGreen(); diff --git a/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreContainerTests.java b/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreContainerTests.java index 7bd24aec8de90..84e7f58cf4935 100644 --- a/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreContainerTests.java +++ b/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreContainerTests.java @@ -19,13 +19,19 @@ package org.elasticsearch.common.blobstore.fs; import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.common.blobstore.BlobContainer; +import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.repositories.ESBlobStoreContainerTestCase; +import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; @LuceneTestCase.SuppressFileSystems("ExtrasFS") public class FsBlobStoreContainerTests extends ESBlobStoreContainerTestCase { @@ -39,4 +45,37 @@ protected BlobStore newBlobStore() throws IOException { } return new FsBlobStore(settings, createTempDir(), false); } + + public void testReadOnly() throws Exception { + Path tempDir = createTempDir(); + Path path = tempDir.resolve("bar"); + + try (FsBlobStore store = new FsBlobStore(Settings.EMPTY, path, true)) { + assertFalse(Files.exists(path)); + BlobPath blobPath = BlobPath.cleanPath().add("foo"); + store.blobContainer(blobPath); + Path storePath = store.path(); + for (String d : blobPath) { + storePath = storePath.resolve(d); + } + assertFalse(Files.exists(storePath)); + } + + try (FsBlobStore store = new FsBlobStore(Settings.EMPTY, path, false)) { + assertTrue(Files.exists(path)); + BlobPath blobPath = BlobPath.cleanPath().add("foo"); + BlobContainer container = store.blobContainer(blobPath); + Path storePath = store.path(); + for (String d : blobPath) { + storePath = storePath.resolve(d); + } + assertTrue(Files.exists(storePath)); + assertTrue(Files.isDirectory(storePath)); + + byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); + writeBlob(container, "test", new BytesArray(data)); + assertArrayEquals(readBlobFully(container, "test", data.length), data); + assertTrue(BlobStoreTestUtil.blobExists(container, "test")); + } + } } diff --git a/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreTests.java b/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreTests.java deleted file mode 100644 index 099d96291adea..0000000000000 --- a/server/src/test/java/org/elasticsearch/common/blobstore/fs/FsBlobStoreTests.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.blobstore.fs; - -import org.apache.lucene.util.LuceneTestCase; -import org.elasticsearch.common.blobstore.BlobContainer; -import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.repositories.ESBlobStoreTestCase; -import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; - -@LuceneTestCase.SuppressFileSystems("ExtrasFS") -public class FsBlobStoreTests extends ESBlobStoreTestCase { - - protected BlobStore newBlobStore() throws IOException { - final Settings settings; - if (randomBoolean()) { - settings = Settings.builder().put("buffer_size", new ByteSizeValue(randomIntBetween(1, 100), ByteSizeUnit.KB)).build(); - } else { - settings = Settings.EMPTY; - } - return new FsBlobStore(settings, createTempDir(), false); - } - - public void testReadOnly() throws Exception { - Path tempDir = createTempDir(); - Path path = tempDir.resolve("bar"); - - try (FsBlobStore store = new FsBlobStore(Settings.EMPTY, path, true)) { - assertFalse(Files.exists(path)); - BlobPath blobPath = BlobPath.cleanPath().add("foo"); - store.blobContainer(blobPath); - Path storePath = store.path(); - for (String d : blobPath) { - storePath = storePath.resolve(d); - } - assertFalse(Files.exists(storePath)); - } - - try (FsBlobStore store = new FsBlobStore(Settings.EMPTY, path, false)) { - assertTrue(Files.exists(path)); - BlobPath blobPath = BlobPath.cleanPath().add("foo"); - BlobContainer container = store.blobContainer(blobPath); - Path storePath = store.path(); - for (String d : blobPath) { - storePath = storePath.resolve(d); - } - assertTrue(Files.exists(storePath)); - assertTrue(Files.isDirectory(storePath)); - - byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); - writeBlob(container, "test", new BytesArray(data)); - assertArrayEquals(readBlobFully(container, "test", data.length), data); - assertTrue(BlobStoreTestUtil.blobExists(container, "test")); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java index 6310bf0457be4..adeb49faa8941 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexModuleTests.java @@ -160,7 +160,7 @@ public void tearDown() throws Exception { private IndexService newIndexService(IndexModule module) throws IOException { return module.newIndexService(CREATE_INDEX, nodeEnvironment, xContentRegistry(), deleter, circuitBreakerService, bigArrays, threadPool, scriptService, clusterService, null, indicesQueryCache, mapperRegistry, - new IndicesFieldDataCache(settings, listener), writableRegistry()); + new IndicesFieldDataCache(settings, listener), writableRegistry(), () -> false); } public void testWrapperIsBound() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index c225b090816ec..fa775a84c72ad 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -94,7 +94,7 @@ private CodecService createCodecService() throws IOException { IndexAnalyzers indexAnalyzers = createTestAnalysis(settings, nodeSettings).indexAnalyzers; MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); MapperService service = new MapperService(settings, indexAnalyzers, xContentRegistry(), similarityService, mapperRegistry, - () -> null); + () -> null, () -> false); return new CodecService(service, LogManager.getLogger("test")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index 5296fd1b89062..4b2d4228ab547 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -78,7 +78,7 @@ public void testGetIndex() { public void testGetFieldMappings() { GetFieldMappingsResponse getFieldMappingsResponse = client().admin().indices().prepareGetFieldMappings().setFields("*").get(); - Map>> mappings = getFieldMappingsResponse.mappings(); + Map> mappings = getFieldMappingsResponse.mappings(); assertEquals(2, mappings.size()); assertFieldMappings(mappings.get("index1"), ALL_FLAT_FIELDS); assertFieldMappings(mappings.get("filtered"), FILTERED_FLAT_FIELDS); @@ -92,6 +92,14 @@ public void testGetFieldMappings() { assertFieldMappings(response.mappings().get("test"), FILTERED_FLAT_FIELDS); } + public void testGetNonExistentFieldMapping() { + GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings("index1").setFields("non-existent").get(); + Map> mappings = response.mappings(); + assertEquals(1, mappings.size()); + Map fieldmapping = mappings.get("index1"); + assertEquals(0, fieldmapping.size()); + } + public void testFieldCapabilities() { List allFields = new ArrayList<>(ALL_FLAT_FIELDS); allFields.addAll(ALL_OBJECT_FIELDS); @@ -126,11 +134,10 @@ private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesR assertEquals("Some unexpected fields were returned: " + responseMap.keySet(), 0, responseMap.size()); } - private static void assertFieldMappings(Map> mappings, + private static void assertFieldMappings(Map actual, Collection expectedFields) { - assertEquals(1, mappings.size()); - Map fields = new HashMap<>(mappings.get("_doc")); Set builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields(); + Map fields = new HashMap<>(actual); for (String field : builtInMetaDataFields) { GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field); assertNotNull(" expected field [" + field + "] not found", fieldMappingMetaData); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index bf6838ec7b67a..3ff6eda1fd7fa 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -41,6 +41,7 @@ import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.geometry.utils.Geohash.stringEncode; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_MALFORMED; import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_Z_VALUE; import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.NULL_VALUE; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; @@ -427,6 +428,32 @@ public void testNullValue() throws Exception { assertThat(defaultValue, not(equalTo(doc.rootDoc().getField("location").binaryValue()))); } + /** + * Test the fix for a bug that would read the value of field "ignore_z_value" for "ignore_malformed" + * when setting the "null_value" field. See PR https://github.com/elastic/elasticsearch/pull/49645 + */ + public void testNullValueWithIgnoreMalformed() throws Exception { + // Set ignore_z_value = false and ignore_malformed = true and test that a malformed point for null_value is normalized. + String mapping = Strings.toString(XContentFactory.jsonBuilder() + .startObject().startObject("type") + .startObject("properties").startObject("location") + .field("type", "geo_point") + .field(IGNORE_Z_VALUE.getPreferredName(), false) + .field(IGNORE_MALFORMED, true) + .field(NULL_VALUE, "91,181") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoPointFieldMapper.class)); + + Object nullValue = ((GeoPointFieldMapper) fieldMapper).fieldType().nullValue(); + // geo_point [91, 181] should have been normalized to [89, 1] + assertThat(nullValue, equalTo(new GeoPoint(89, 1))); + } + public void testInvalidGeohashIgnored() throws Exception { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("properties") diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java index 4f2198c577a3f..f495da0186ab6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdFieldMapperTests.java @@ -28,7 +28,9 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -37,6 +39,9 @@ import java.util.Collection; import java.util.Collections; +import static org.elasticsearch.index.mapper.IdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE; +import static org.hamcrest.Matchers.containsString; + public class IdFieldMapperTests extends ESSingleNodeTestCase { @Override @@ -71,4 +76,30 @@ public void testDefaults() throws IOException { assertEquals(Uid.encodeId("id"), fields[0].binaryValue()); } + public void testEnableFieldData() throws IOException { + IndexService service = createIndex("test", Settings.EMPTY); + MapperService mapperService = service.mapperService(); + mapperService.merge("type", new CompressedXContent("{\"type\":{}}"), MergeReason.MAPPING_UPDATE); + IdFieldMapper.IdFieldType ft = (IdFieldMapper.IdFieldType) service.mapperService().fullName("_id"); + + IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, + () -> ft.fielddataBuilder("test").build(mapperService.getIndexSettings(), + ft, null, null, mapperService)); + assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); + + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true)) + .get(); + try { + ft.fielddataBuilder("test").build(mapperService.getIndexSettings(), + ft, null, null, mapperService); + assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE); + } finally { + // unset cluster setting + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())) + .get(); + } + } + } diff --git a/server/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java b/server/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java index 6bf89ee76642d..f4213eb486f8d 100644 --- a/server/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java +++ b/server/src/test/java/org/elasticsearch/indices/mapping/SimpleGetFieldMappingsIT.java @@ -50,7 +50,6 @@ import static org.hamcrest.Matchers.hasEntry; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.nullValue; public class SimpleGetFieldMappingsIT extends ESIntegTestCase { @@ -65,7 +64,7 @@ public void testGetMappingsWhereThereAreNone() { assertThat(response.mappings().size(), equalTo(1)); assertThat(response.mappings().get("index").size(), equalTo(0)); - assertThat(response.fieldMappings("index", "type", "field"), Matchers.nullValue()); + assertThat(response.fieldMappings("index", "field"), Matchers.nullValue()); } private XContentBuilder getMappingForType(String type) throws IOException { @@ -100,30 +99,28 @@ public void testGetFieldMappings() throws Exception { // Get mappings by full name - GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings("indexa").setTypes("_doc") + GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings("indexa") .setFields("field1", "obj.subfield").get(); - assertThat(response.fieldMappings("indexa", "_doc", "field1").fullName(), equalTo("field1")); - assertThat(response.fieldMappings("indexa", "_doc", "field1").sourceAsMap(), hasKey("field1")); - assertThat(response.fieldMappings("indexa", "_doc", "obj.subfield").fullName(), equalTo("obj.subfield")); - assertThat(response.fieldMappings("indexa", "_doc", "obj.subfield").sourceAsMap(), hasKey("subfield")); - assertThat(response.fieldMappings("indexb", "typeB", "field1"), nullValue()); + assertThat(response.fieldMappings("indexa", "field1").fullName(), equalTo("field1")); + assertThat(response.fieldMappings("indexa", "field1").sourceAsMap(), hasKey("field1")); + assertThat(response.fieldMappings("indexa", "obj.subfield").fullName(), equalTo("obj.subfield")); + assertThat(response.fieldMappings("indexa", "obj.subfield").sourceAsMap(), hasKey("subfield")); // Get mappings by name - response = client().admin().indices().prepareGetFieldMappings("indexa").setTypes("_doc").setFields("field1", "obj.subfield") + response = client().admin().indices().prepareGetFieldMappings("indexa").setFields("field1", "obj.subfield") .get(); - assertThat(response.fieldMappings("indexa", "_doc", "field1").fullName(), equalTo("field1")); - assertThat(response.fieldMappings("indexa", "_doc", "field1").sourceAsMap(), hasKey("field1")); - assertThat(response.fieldMappings("indexa", "_doc", "obj.subfield").fullName(), equalTo("obj.subfield")); - assertThat(response.fieldMappings("indexa", "_doc", "obj.subfield").sourceAsMap(), hasKey("subfield")); - assertThat(response.fieldMappings("indexa", "typeB", "field1"), nullValue()); - assertThat(response.fieldMappings("indexb", "typeB", "field1"), nullValue()); + assertThat(response.fieldMappings("indexa", "field1").fullName(), equalTo("field1")); + assertThat(response.fieldMappings("indexa", "field1").sourceAsMap(), hasKey("field1")); + assertThat(response.fieldMappings("indexa", "obj.subfield").fullName(), equalTo("obj.subfield")); + assertThat(response.fieldMappings("indexa", "obj.subfield").sourceAsMap(), hasKey("subfield")); // get mappings by name across multiple indices - response = client().admin().indices().prepareGetFieldMappings().setTypes("_doc").setFields("obj.subfield").get(); - assertThat(response.fieldMappings("indexa", "_doc", "obj.subfield").fullName(), equalTo("obj.subfield")); - assertThat(response.fieldMappings("indexa", "_doc", "obj.subfield").sourceAsMap(), hasKey("subfield")); - assertThat(response.fieldMappings("indexa", "typeB", "obj.subfield"), nullValue()); - assertThat(response.fieldMappings("indexb", "typeB", "obj.subfield"), nullValue()); + response = client().admin().indices().prepareGetFieldMappings().setFields("obj.subfield").get(); + assertThat(response.fieldMappings("indexa", "obj.subfield").fullName(), equalTo("obj.subfield")); + assertThat(response.fieldMappings("indexa", "obj.subfield").sourceAsMap(), hasKey("subfield")); + assertThat(response.fieldMappings("indexb", "obj.subfield").fullName(), equalTo("obj.subfield")); + assertThat(response.fieldMappings("indexb", "obj.subfield").sourceAsMap(), hasKey("subfield")); + } @SuppressWarnings("unchecked") @@ -134,15 +131,15 @@ public void testSimpleGetFieldMappingsWithDefaults() throws Exception { GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings() .setFields("num", "field1", "obj.subfield").includeDefaults(true).get(); - assertThat((Map) response.fieldMappings("test", "_doc", "num").sourceAsMap().get("num"), + assertThat((Map) response.fieldMappings("test", "num").sourceAsMap().get("num"), hasEntry("index", Boolean.TRUE)); - assertThat((Map) response.fieldMappings("test", "_doc", "num").sourceAsMap().get("num"), + assertThat((Map) response.fieldMappings("test", "num").sourceAsMap().get("num"), hasEntry("type", "long")); - assertThat((Map) response.fieldMappings("test", "_doc", "field1").sourceAsMap().get("field1"), + assertThat((Map) response.fieldMappings("test", "field1").sourceAsMap().get("field1"), hasEntry("index", Boolean.TRUE)); - assertThat((Map) response.fieldMappings("test", "_doc", "field1").sourceAsMap().get("field1"), + assertThat((Map) response.fieldMappings("test", "field1").sourceAsMap().get("field1"), hasEntry("type", "text")); - assertThat((Map) response.fieldMappings("test", "_doc", "obj.subfield").sourceAsMap().get("subfield"), + assertThat((Map) response.fieldMappings("test", "obj.subfield").sourceAsMap().get("subfield"), hasEntry("type", "keyword")); } @@ -153,12 +150,12 @@ public void testGetFieldMappingsWithFieldAlias() throws Exception { GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings() .setFields("alias", "field1").get(); - FieldMappingMetaData aliasMapping = response.fieldMappings("test", "_doc", "alias"); + FieldMappingMetaData aliasMapping = response.fieldMappings("test", "alias"); assertThat(aliasMapping.fullName(), equalTo("alias")); assertThat(aliasMapping.sourceAsMap(), hasKey("alias")); assertThat((Map) aliasMapping.sourceAsMap().get("alias"), hasEntry("type", "alias")); - FieldMappingMetaData field1Mapping = response.fieldMappings("test", "_doc", "field1"); + FieldMappingMetaData field1Mapping = response.fieldMappings("test", "field1"); assertThat(field1Mapping.fullName(), equalTo("field1")); assertThat(field1Mapping.sourceAsMap(), hasKey("field1")); } @@ -169,7 +166,7 @@ public void testSimpleGetFieldMappingsWithPretty() throws Exception { Map params = new HashMap<>(); params.put("pretty", "true"); GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings("index") - .setTypes("type").setFields("field1", "obj.subfield").get(); + .setFields("field1", "obj.subfield").get(); XContentBuilder responseBuilder = XContentFactory.jsonBuilder().prettyPrint(); response.toXContent(responseBuilder, new ToXContent.MapParams(params)); String responseStrings = Strings.toString(responseBuilder); @@ -182,7 +179,7 @@ public void testSimpleGetFieldMappingsWithPretty() throws Exception { params.put("pretty", "false"); response = client().admin().indices().prepareGetFieldMappings("index") - .setTypes("type").setFields("field1", "obj.subfield").get(); + .setFields("field1", "obj.subfield").get(); responseBuilder = XContentFactory.jsonBuilder().prettyPrint().lfAtEnd(); response.toXContent(responseBuilder, new ToXContent.MapParams(params)); responseStrings = Strings.toString(responseBuilder); @@ -200,9 +197,9 @@ public void testGetFieldMappingsWithBlocks() throws Exception { for (String block : Arrays.asList(SETTING_BLOCKS_READ, SETTING_BLOCKS_WRITE, SETTING_READ_ONLY)) { try { enableIndexBlock("test", block); - GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings("test").setTypes("_doc") + GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings("test") .setFields("field1", "obj.subfield").get(); - assertThat(response.fieldMappings("test", "_doc", "field1").fullName(), equalTo("field1")); + assertThat(response.fieldMappings("test", "field1").fullName(), equalTo("field1")); } finally { disableIndexBlock("test", block); } diff --git a/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java index b3b8ee9762dc1..377679afc43f7 100644 --- a/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/CompoundProcessorTests.java @@ -26,14 +26,17 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.function.LongSupplier; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; import static org.mockito.Mockito.mock; @@ -279,6 +282,82 @@ public void testBreakOnFailure() throws Exception { assertStats(pipeline, 1, 1, 0); } + public void testFailureProcessorIsInvokedOnFailure() { + TestProcessor onFailureProcessor = new TestProcessor(null, "on_failure", ingestDocument -> { + Map ingestMetadata = ingestDocument.getIngestMetadata(); + assertThat(ingestMetadata.entrySet(), hasSize(4)); + assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_MESSAGE_FIELD), equalTo("failure!")); + assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD), equalTo("test-processor")); + assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD), nullValue()); + assertThat(ingestMetadata.get(CompoundProcessor.ON_FAILURE_PIPELINE_FIELD), equalTo("2")); + }); + + Pipeline pipeline2 = new Pipeline("2", null, null, new CompoundProcessor(new TestProcessor(new RuntimeException("failure!")))); + Pipeline pipeline1 = new Pipeline("1", null, null, new CompoundProcessor(false, List.of(new AbstractProcessor(null) { + @Override + public void execute(IngestDocument ingestDocument, BiConsumer handler) { + ingestDocument.executePipeline(pipeline2, handler); + } + + @Override + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + throw new AssertionError(); + } + + @Override + public String getType() { + return "pipeline"; + } + }), List.of(onFailureProcessor))); + + ingestDocument.executePipeline(pipeline1, (document, e) -> { + assertThat(document, notNullValue()); + assertThat(e, nullValue()); + }); + assertThat(onFailureProcessor.getInvokedCounter(), equalTo(1)); + } + + public void testNewCompoundProcessorException() { + TestProcessor processor = new TestProcessor("my_tag", "my_type", new RuntimeException()); + IngestProcessorException ingestProcessorException1 = + CompoundProcessor.newCompoundProcessorException(new RuntimeException(), processor, ingestDocument); + assertThat(ingestProcessorException1.getHeader("processor_tag"), equalTo(List.of("my_tag"))); + assertThat(ingestProcessorException1.getHeader("processor_type"), equalTo(List.of("my_type"))); + assertThat(ingestProcessorException1.getHeader("pipeline_origin"), nullValue()); + + IngestProcessorException ingestProcessorException2 = + CompoundProcessor.newCompoundProcessorException(ingestProcessorException1, processor, ingestDocument); + assertThat(ingestProcessorException2, sameInstance(ingestProcessorException1)); + } + + public void testNewCompoundProcessorExceptionPipelineOrigin() { + Pipeline pipeline2 = new Pipeline("2", null, null, + new CompoundProcessor(new TestProcessor("my_tag", "my_type", new RuntimeException()))); + Pipeline pipeline1 = new Pipeline("1", null, null, new CompoundProcessor(new AbstractProcessor(null) { + @Override + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { + throw new UnsupportedOperationException(); + } + + @Override + public void execute(IngestDocument ingestDocument, BiConsumer handler) { + ingestDocument.executePipeline(pipeline2, handler); + } + + @Override + public String getType() { + return "my_type2"; + } + })); + + Exception[] holder = new Exception[1]; + ingestDocument.executePipeline(pipeline1, (document, e) -> holder[0] = e); + IngestProcessorException ingestProcessorException = (IngestProcessorException) holder[0]; + assertThat(ingestProcessorException.getHeader("processor_tag"), equalTo(List.of("my_tag"))); + assertThat(ingestProcessorException.getHeader("processor_type"), equalTo(List.of("my_type"))); + assertThat(ingestProcessorException.getHeader("pipeline_origin"), equalTo(List.of("2", "1"))); + } + private void assertStats(CompoundProcessor compoundProcessor, long count, long failed, long time) { assertStats(0, compoundProcessor, 0L, count, failed, time); } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java b/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java index 762f2ba5eb937..512962325a27c 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestClientIT.java @@ -39,13 +39,14 @@ import org.elasticsearch.client.Requests; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -68,7 +69,7 @@ protected Settings nodeSettings(int nodeOrdinal) { @Override protected Collection> nodePlugins() { - return Arrays.asList(IngestTestPlugin.class); + return List.of(ExtendedIngestTestPlugin.class); } public void testSimulate() throws Exception { @@ -293,4 +294,157 @@ public void testWithDedicatedMaster() throws Exception { assertFalse(item.isFailed()); assertEquals("auto-generated", item.getResponse().getId()); } + + public void testPipelineOriginHeader() throws Exception { + { + XContentBuilder source = jsonBuilder().startObject(); + { + source.startArray("processors"); + source.startObject(); + { + source.startObject("pipeline"); + source.field("name", "2"); + source.endObject(); + } + source.endObject(); + source.endArray(); + } + source.endObject(); + PutPipelineRequest putPipelineRequest = + new PutPipelineRequest("1", BytesReference.bytes(source), XContentType.JSON); + client().admin().cluster().putPipeline(putPipelineRequest).get(); + } + { + XContentBuilder source = jsonBuilder().startObject(); + { + source.startArray("processors"); + source.startObject(); + { + source.startObject("pipeline"); + source.field("name", "3"); + source.endObject(); + } + source.endObject(); + source.endArray(); + } + source.endObject(); + PutPipelineRequest putPipelineRequest = + new PutPipelineRequest("2", BytesReference.bytes(source), XContentType.JSON); + client().admin().cluster().putPipeline(putPipelineRequest).get(); + } + { + XContentBuilder source = jsonBuilder().startObject(); + { + source.startArray("processors"); + source.startObject(); + { + source.startObject("fail"); + source.endObject(); + } + source.endObject(); + source.endArray(); + } + source.endObject(); + PutPipelineRequest putPipelineRequest = + new PutPipelineRequest("3", BytesReference.bytes(source), XContentType.JSON); + client().admin().cluster().putPipeline(putPipelineRequest).get(); + } + + Exception e = expectThrows(Exception.class, () -> { + IndexRequest indexRequest = new IndexRequest("test"); + indexRequest.source("{}", XContentType.JSON); + indexRequest.setPipeline("1"); + client().index(indexRequest).get(); + }); + IngestProcessorException ingestException = (IngestProcessorException) e.getCause(); + assertThat(ingestException.getHeader("processor_type"), equalTo(List.of("fail"))); + assertThat(ingestException.getHeader("pipeline_origin"), equalTo(List.of("3", "2", "1"))); + } + + public void testPipelineProcessorOnFailure() throws Exception { + { + XContentBuilder source = jsonBuilder().startObject(); + { + source.startArray("processors"); + source.startObject(); + { + source.startObject("pipeline"); + source.field("name", "2"); + source.endObject(); + } + source.endObject(); + source.endArray(); + } + { + source.startArray("on_failure"); + source.startObject(); + { + source.startObject("onfailure_processor"); + source.endObject(); + } + source.endObject(); + source.endArray(); + } + source.endObject(); + PutPipelineRequest putPipelineRequest = + new PutPipelineRequest("1", BytesReference.bytes(source), XContentType.JSON); + client().admin().cluster().putPipeline(putPipelineRequest).get(); + } + { + XContentBuilder source = jsonBuilder().startObject(); + { + source.startArray("processors"); + source.startObject(); + { + source.startObject("pipeline"); + source.field("name", "3"); + source.endObject(); + } + source.endObject(); + source.endArray(); + } + source.endObject(); + PutPipelineRequest putPipelineRequest = + new PutPipelineRequest("2", BytesReference.bytes(source), XContentType.JSON); + client().admin().cluster().putPipeline(putPipelineRequest).get(); + } + { + XContentBuilder source = jsonBuilder().startObject(); + { + source.startArray("processors"); + source.startObject(); + { + source.startObject("fail"); + source.endObject(); + } + source.endObject(); + source.endArray(); + } + source.endObject(); + PutPipelineRequest putPipelineRequest = + new PutPipelineRequest("3", BytesReference.bytes(source), XContentType.JSON); + client().admin().cluster().putPipeline(putPipelineRequest).get(); + } + + client().prepareIndex("test").setId("1").setSource("{}", XContentType.JSON).setPipeline("1").get(); + Map inserted = client().prepareGet("test", "1") + .get().getSourceAsMap(); + assertThat(inserted.get("readme"), equalTo("pipeline with id [3] is a bad pipeline")); + } + + public static class ExtendedIngestTestPlugin extends IngestTestPlugin { + + @Override + public Map getProcessors(Processor.Parameters parameters) { + Map factories = new HashMap<>(super.getProcessors(parameters)); + factories.put(PipelineProcessor.TYPE, new PipelineProcessor.Factory(parameters.ingestService)); + factories.put("fail", (processorFactories, tag, config) -> new TestProcessor(tag, "fail", new RuntimeException())); + factories.put("onfailure_processor", (processorFactories, tag, config) -> new TestProcessor(tag, "fail", document -> { + String onFailurePipeline = document.getFieldValue("_ingest.on_failure_pipeline", String.class); + document.setFieldValue("readme", "pipeline with id [" + onFailurePipeline + "] is a bad pipeline"); + })); + return factories; + } + } + } diff --git a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java index 93b1589617ea7..5400956d076c3 100644 --- a/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/IngestServiceTests.java @@ -1115,7 +1115,7 @@ public void testStatName(){ PipelineProcessor pipelineProcessor = mock(PipelineProcessor.class); String pipelineName = randomAlphaOfLength(10); - when(pipelineProcessor.getPipelineName()).thenReturn(pipelineName); + when(pipelineProcessor.getPipelineTemplate()).thenReturn(new TestTemplateService.MockTemplateScript.Factory(pipelineName)); name = PipelineProcessor.TYPE; when(pipelineProcessor.getType()).thenReturn(name); assertThat(IngestService.getProcessorName(pipelineProcessor), equalTo(name + ":" + pipelineName)); diff --git a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java index 4f36727c7ac30..aebcc28e77d5e 100644 --- a/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/PipelineProcessorTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.ingest; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import java.util.Arrays; @@ -37,7 +38,7 @@ public class PipelineProcessorTests extends ESTestCase { public void testExecutesPipeline() throws Exception { String pipelineId = "pipeline"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); CompletableFuture invoked = new CompletableFuture<>(); IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); Pipeline pipeline = new Pipeline( @@ -69,7 +70,7 @@ public String getTag() { } public void testThrowsOnMissingPipeline() throws Exception { - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); Map config = new HashMap<>(); @@ -85,7 +86,7 @@ public void testThrowsOnMissingPipeline() throws Exception { public void testThrowsOnRecursivePipelineInvocations() throws Exception { String innerPipelineId = "inner"; String outerPipelineId = "outer"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); Map outerConfig = new HashMap<>(); outerConfig.put("name", innerPipelineId); @@ -113,7 +114,7 @@ public void testThrowsOnRecursivePipelineInvocations() throws Exception { public void testAllowsRepeatedPipelineInvocations() throws Exception { String innerPipelineId = "inner"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); IngestDocument testIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); Map outerConfig = new HashMap<>(); outerConfig.put("name", innerPipelineId); @@ -131,7 +132,7 @@ public void testPipelineProcessorWithPipelineChain() throws Exception { String pipeline1Id = "pipeline1"; String pipeline2Id = "pipeline2"; String pipeline3Id = "pipeline3"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); Map pipeline1ProcessorConfig = new HashMap<>(); @@ -203,4 +204,11 @@ pipeline3Id, null, null, new CompoundProcessor( assertThat(pipeline2Stats.getIngestFailedCount(), equalTo(0L)); assertThat(pipeline3Stats.getIngestFailedCount(), equalTo(1L)); } + + static IngestService createIngestService() { + IngestService ingestService = mock(IngestService.class); + ScriptService scriptService = mock(ScriptService.class); + when(ingestService.getScriptService()).thenReturn(scriptService); + return ingestService; + } } diff --git a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java index cc9e44e387baf..c66d4742b991b 100644 --- a/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java +++ b/server/src/test/java/org/elasticsearch/ingest/TrackingResultProcessorTests.java @@ -40,6 +40,7 @@ import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_MESSAGE_FIELD; import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_PROCESSOR_TAG_FIELD; import static org.elasticsearch.ingest.CompoundProcessor.ON_FAILURE_PROCESSOR_TYPE_FIELD; +import static org.elasticsearch.ingest.PipelineProcessorTests.createIngestService; import static org.elasticsearch.ingest.TrackingResultProcessor.decorate; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.not; @@ -47,7 +48,6 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.sameInstance; -import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -195,7 +195,7 @@ public void testActualCompoundProcessorWithFalseConditional() throws Exception { public void testActualPipelineProcessor() throws Exception { String pipelineId = "pipeline1"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); Map pipelineConfig = new HashMap<>(); pipelineConfig.put("name", pipelineId); PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); @@ -240,7 +240,7 @@ pipelineId, null, null, new CompoundProcessor( public void testActualPipelineProcessorWithTrueConditional() throws Exception { String pipelineId1 = "pipeline1"; String pipelineId2 = "pipeline2"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); Map pipelineConfig0 = new HashMap<>(); pipelineConfig0.put("name", pipelineId1); Map pipelineConfig1 = new HashMap<>(); @@ -308,7 +308,7 @@ pipelineId2, null, null, new CompoundProcessor( public void testActualPipelineProcessorWithFalseConditional() throws Exception { String pipelineId1 = "pipeline1"; String pipelineId2 = "pipeline2"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); Map pipelineConfig0 = new HashMap<>(); pipelineConfig0.put("name", pipelineId1); Map pipelineConfig1 = new HashMap<>(); @@ -377,7 +377,7 @@ public void testActualPipelineProcessorWithHandledFailure() throws Exception { RuntimeException exception = new RuntimeException("processor failed"); String pipelineId = "pipeline1"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); Map pipelineConfig = new HashMap<>(); pipelineConfig.put("name", pipelineId); PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); @@ -430,7 +430,7 @@ pipelineId, null, null, new CompoundProcessor( public void testActualPipelineProcessorWithCycle() throws Exception { String pipelineId1 = "pipeline1"; String pipelineId2 = "pipeline2"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); Map pipelineConfig0 = new HashMap<>(); pipelineConfig0.put("name", pipelineId1); Map pipelineConfig1 = new HashMap<>(); @@ -462,7 +462,7 @@ public void testActualPipelineProcessorWithCycle() throws Exception { public void testActualPipelineProcessorRepeatedInvocation() throws Exception { String pipelineId = "pipeline1"; - IngestService ingestService = mock(IngestService.class); + IngestService ingestService = createIngestService(); Map pipelineConfig = new HashMap<>(); pipelineConfig.put("name", pipelineId); PipelineProcessor.Factory factory = new PipelineProcessor.Factory(ingestService); diff --git a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java index c100c582df793..425ccdfe8ccc2 100644 --- a/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java +++ b/server/src/test/java/org/elasticsearch/repositories/blobstore/BlobStoreRepositoryTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.repositories.blobstore; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.UUIDs; @@ -42,7 +43,6 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.threadpool.ThreadPool; -import java.io.IOException; import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; @@ -141,7 +141,7 @@ public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception { // write to and read from a index file with no entries assertThat(ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository).getSnapshotIds().size(), equalTo(0)); final RepositoryData emptyData = RepositoryData.EMPTY; - repository.writeIndexGen(emptyData, emptyData.getGenId(), true); + writeIndexGen(repository, emptyData, emptyData.getGenId()); RepositoryData repoData = ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository); assertEquals(repoData, emptyData); assertEquals(repoData.getIndices().size(), 0); @@ -150,28 +150,29 @@ public void testReadAndWriteSnapshotsThroughIndexFile() throws Exception { // write to and read from an index file with snapshots but no indices repoData = addRandomSnapshotsToRepoData(repoData, false); - repository.writeIndexGen(repoData, repoData.getGenId(), true); + writeIndexGen(repository, repoData, repoData.getGenId()); assertEquals(repoData, ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository)); // write to and read from a index file with random repository data repoData = addRandomSnapshotsToRepoData(ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), true); - repository.writeIndexGen(repoData, repoData.getGenId(), true); + writeIndexGen(repository, repoData, repoData.getGenId()); assertEquals(repoData, ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository)); } public void testIndexGenerationalFiles() throws Exception { final BlobStoreRepository repository = setupRepo(); + assertEquals(ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), RepositoryData.EMPTY); // write to index generational file RepositoryData repositoryData = generateRandomRepoData(); - repository.writeIndexGen(repositoryData, repositoryData.getGenId(), true); + writeIndexGen(repository, repositoryData, RepositoryData.EMPTY_REPO_GEN); assertThat(ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), equalTo(repositoryData)); assertThat(repository.latestIndexBlobId(), equalTo(0L)); assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(0L)); // adding more and writing to a new index generational file repositoryData = addRandomSnapshotsToRepoData(ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), true); - repository.writeIndexGen(repositoryData, repositoryData.getGenId(), true); + writeIndexGen(repository, repositoryData, repositoryData.getGenId()); assertEquals(ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), repositoryData); assertThat(repository.latestIndexBlobId(), equalTo(1L)); assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(1L)); @@ -179,24 +180,25 @@ public void testIndexGenerationalFiles() throws Exception { // removing a snapshot and writing to a new index generational file repositoryData = ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository).removeSnapshot( repositoryData.getSnapshotIds().iterator().next(), ShardGenerations.EMPTY); - repository.writeIndexGen(repositoryData, repositoryData.getGenId(), true); + writeIndexGen(repository, repositoryData, repositoryData.getGenId()); assertEquals(ESBlobStoreRepositoryIntegTestCase.getRepositoryData(repository), repositoryData); assertThat(repository.latestIndexBlobId(), equalTo(2L)); assertThat(repository.readSnapshotIndexLatestBlob(), equalTo(2L)); } - public void testRepositoryDataConcurrentModificationNotAllowed() throws IOException { + public void testRepositoryDataConcurrentModificationNotAllowed() { final BlobStoreRepository repository = setupRepo(); // write to index generational file RepositoryData repositoryData = generateRandomRepoData(); final long startingGeneration = repositoryData.getGenId(); - repository.writeIndexGen(repositoryData, startingGeneration, true); + final PlainActionFuture future1 = PlainActionFuture.newFuture(); + repository.writeIndexGen(repositoryData, startingGeneration, true, future1); // write repo data again to index generational file, errors because we already wrote to the // N+1 generation from which this repository data instance was created - expectThrows(RepositoryException.class, () -> repository.writeIndexGen( - repositoryData.withGenId(startingGeneration + 1), repositoryData.getGenId(), true)); + expectThrows(RepositoryException.class, + () -> writeIndexGen(repository, repositoryData.withGenId(startingGeneration + 1), repositoryData.getGenId())); } public void testBadChunksize() throws Exception { @@ -213,6 +215,12 @@ public void testBadChunksize() throws Exception { .get()); } + private static void writeIndexGen(BlobStoreRepository repository, RepositoryData repositoryData, long generation) { + final PlainActionFuture future = PlainActionFuture.newFuture(); + repository.writeIndexGen(repositoryData, generation, true, future); + future.actionGet(); + } + private BlobStoreRepository setupRepo() { final Client client = client(); final Path location = ESIntegTestCase.randomRepoPath(node().settings()); diff --git a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java index eba4a03e72cfa..cdbe140b0f83c 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchCancellationTests.java @@ -24,12 +24,13 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.store.Directory; import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.TestUtil; -import org.elasticsearch.search.query.CancellableCollector; +import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.test.ESTestCase; import org.junit.AfterClass; @@ -38,6 +39,8 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.equalTo; + public class SearchCancellationTests extends ESTestCase { static Directory dir; @@ -75,12 +78,18 @@ public static void cleanup() throws IOException { public void testCancellableCollector() throws IOException { TotalHitCountCollector collector = new TotalHitCountCollector(); AtomicBoolean cancelled = new AtomicBoolean(); - CancellableCollector cancellableCollector = new CancellableCollector(cancelled::get, collector); - final LeafCollector leafCollector = cancellableCollector.getLeafCollector(reader.leaves().get(0)); - leafCollector.collect(0); + ContextIndexSearcher searcher = new ContextIndexSearcher(reader, + IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); + searcher.setCheckCancelled(() -> { + if (cancelled.get()) { + throw new TaskCancelledException("cancelled"); + } + }); + searcher.search(new MatchAllDocsQuery(), collector); + assertThat(collector.getTotalHits(), equalTo(reader.numDocs())); cancelled.set(true); - leafCollector.collect(1); - expectThrows(TaskCancelledException.class, () -> cancellableCollector.getLeafCollector(reader.leaves().get(1))); + expectThrows(TaskCancelledException.class, + () -> searcher.search(new MatchAllDocsQuery(), collector)); } } diff --git a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java index d272e0603240a..fc991ca43cbc9 100644 --- a/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java +++ b/server/src/test/java/org/elasticsearch/search/profile/query/QueryProfilerIT.java @@ -119,6 +119,7 @@ public void testProfileMatchesRegular() throws Exception { IndexRequestBuilder[] docs = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; i++) { docs[i] = client().prepareIndex("test").setId(String.valueOf(i)).setSource( + "id", String.valueOf(i), "field1", English.intToEnglish(i), "field2", i ); @@ -136,14 +137,14 @@ public void testProfileMatchesRegular() throws Exception { SearchRequestBuilder vanilla = client().prepareSearch("test") .setQuery(q) .setProfile(false) - .addSort("_id", SortOrder.ASC) + .addSort("id.keyword", SortOrder.ASC) .setSearchType(SearchType.QUERY_THEN_FETCH) .setRequestCache(false); SearchRequestBuilder profile = client().prepareSearch("test") .setQuery(q) .setProfile(true) - .addSort("_id", SortOrder.ASC) + .addSort("id.keyword", SortOrder.ASC) .setSearchType(SearchType.QUERY_THEN_FETCH) .setRequestCache(false); diff --git a/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java index cb0b448ede879..a15c33832b8df 100644 --- a/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/MultiMatchQueryIT.java @@ -97,6 +97,7 @@ public void init() throws Exception { int numDocs = scaledRandomIntBetween(50, 100); List builders = new ArrayList<>(); builders.add(client().prepareIndex("test").setId("theone").setSource( + "id", "theone", "full_name", "Captain America", "first_name", "Captain", "last_name", "America", @@ -104,6 +105,7 @@ public void init() throws Exception { "skill", 15, "int-field", 25)); builders.add(client().prepareIndex("test").setId("theother").setSource( + "id", "theother", "full_name", "marvel hero", "first_name", "marvel", "last_name", "hero", @@ -111,6 +113,7 @@ public void init() throws Exception { "skill", 5)); builders.add(client().prepareIndex("test").setId("ultimate1").setSource( + "id", "ultimate1", "full_name", "Alpha the Ultimate Mutant", "first_name", "Alpha the", "last_name", "Ultimate Mutant", @@ -124,6 +127,7 @@ public void init() throws Exception { "skill", 3)); builders.add(client().prepareIndex("test").setId("anotherhero").setSource( + "id", "anotherhero", "full_name", "ultimate", "first_name", "wolferine", "last_name", "", @@ -131,6 +135,7 @@ public void init() throws Exception { "skill", 1)); builders.add(client().prepareIndex("test").setId("nowHero").setSource( + "id", "nowHero", "full_name", "now sort of", "first_name", "now", "last_name", "", @@ -147,6 +152,7 @@ public void init() throws Exception { String first = RandomPicks.randomFrom(random(), firstNames); String last = randomPickExcept(lastNames, first); builders.add(client().prepareIndex("test").setId("" + i).setSource( + "id", i, "full_name", first + " " + last, "first_name", first, "last_name", last, @@ -159,6 +165,9 @@ public void init() throws Exception { private XContentBuilder createMapping() throws IOException { return XContentFactory.jsonBuilder().startObject().startObject("test") .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("full_name") .field("type", "text") .field("copy_to", "full_name_phrase") @@ -274,17 +283,17 @@ public void testSingleField() throws NoSuchFieldException, IllegalAccessExceptio } MultiMatchQueryBuilder multiMatchQueryBuilder = randomizeType(multiMatchQuery(builder.toString(), field)); SearchResponse multiMatchResp = client().prepareSearch("test") - // _id sort field is a tie, in case hits have the same score, + // id sort field is a tie, in case hits have the same score, // the hits will be sorted the same consistently .addSort("_score", SortOrder.DESC) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(multiMatchQueryBuilder).get(); MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery(field, builder.toString()); SearchResponse matchResp = client().prepareSearch("test") - // _id tie sort + // id tie sort .addSort("_score", SortOrder.DESC) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(matchQueryBuilder).get(); assertThat("field: " + field + " query: " + builder.toString(), multiMatchResp.getHits().getTotalHits().value, equalTo(matchResp.getHits().getTotalHits().value)); @@ -312,12 +321,12 @@ public void testEquivalence() { multiMatchQuery("marvel hero captain america", "full_name", "first_name", "last_name", "category") : multiMatchQuery("marvel hero captain america", "*_name", randomBoolean() ? "category" : "categ*"); SearchResponse left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQueryBuilder .operator(Operator.OR).type(type))).get(); SearchResponse right = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(disMaxQuery(). add(matchQuery("full_name", "marvel hero captain america")) .add(matchQuery("first_name", "marvel hero captain america")) @@ -335,12 +344,12 @@ public void testEquivalence() { multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category") : multiMatchQuery("captain america", "*_name", randomBoolean() ? "category" : "categ*"); SearchResponse left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQueryBuilder .operator(op).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch).type(type))).get(); SearchResponse right = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(boolQuery().minimumShouldMatch(minShouldMatch) .should(randomBoolean() ? termQuery("full_name", "captain america") : matchQuery("full_name", "captain america").operator(op)) @@ -354,12 +363,12 @@ public void testEquivalence() { { String minShouldMatch = randomBoolean() ? null : "" + between(0, 1); SearchResponse left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQuery("capta", "full_name", "first_name", "last_name", "category") .type(MatchQuery.Type.PHRASE_PREFIX).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch))).get(); SearchResponse right = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(boolQuery().minimumShouldMatch(minShouldMatch) .should(matchPhrasePrefixQuery("full_name", "capta")) .should(matchPhrasePrefixQuery("first_name", "capta")) @@ -373,17 +382,17 @@ public void testEquivalence() { SearchResponse left; if (randomBoolean()) { left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category") .type(MatchQuery.Type.PHRASE).minimumShouldMatch(minShouldMatch))).get(); } else { left = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(randomizeType(multiMatchQuery("captain america", "full_name", "first_name", "last_name", "category") .type(MatchQuery.Type.PHRASE).tieBreaker(1.0f).minimumShouldMatch(minShouldMatch))).get(); } SearchResponse right = client().prepareSearch("test").setSize(numDocs) - .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("_id")) + .addSort(SortBuilders.scoreSort()).addSort(SortBuilders.fieldSort("id")) .setQuery(boolQuery().minimumShouldMatch(minShouldMatch) .should(matchPhraseQuery("full_name", "captain america")) .should(matchPhraseQuery("first_name", "captain america")) diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 2190e573707e6..3f83848620462 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -24,21 +24,25 @@ import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.LatLonDocValuesField; import org.apache.lucene.document.LatLonPoint; +import org.apache.lucene.document.LongPoint; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queries.MinDocQuery; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; +import org.apache.lucene.search.CollectorManager; import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.FieldComparator; @@ -50,9 +54,11 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TotalHitCountCollector; import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.Weight; @@ -65,11 +71,16 @@ import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.action.search.SearchTask; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.search.ESToParentBlockJoinQuery; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.internal.ContextIndexSearcher; import org.elasticsearch.search.internal.ScrollContext; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.SortAndFormats; @@ -80,10 +91,15 @@ import java.util.Collections; import java.util.List; +import static org.elasticsearch.search.query.QueryPhase.indexFieldHasDuplicateData; +import static org.elasticsearch.search.query.TopDocsCollectorContext.hasInfMaxScore; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.spy; public class QueryPhaseTests extends IndexShardTestCase { @@ -107,18 +123,17 @@ public void tearDown() throws Exception { } private void countTestCase(Query query, IndexReader reader, boolean shouldCollectSearch, boolean shouldCollectCount) throws Exception { - TestSearchContext context = new TestSearchContext(null, indexShard); + ContextIndexSearcher searcher = shouldCollectSearch ? newContextSearcher(reader) : + newEarlyTerminationContextSearcher(reader, 0); + TestSearchContext context = new TestSearchContext(null, indexShard, searcher); context.parsedQuery(new ParsedQuery(query)); context.setSize(0); context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); - - final IndexSearcher searcher = shouldCollectSearch ? new IndexSearcher(reader) : - getAssertingEarlyTerminationSearcher(reader, 0); - - final boolean rescore = QueryPhase.execute(context, searcher, checkCancelled -> {}); + final boolean rescore = QueryPhase.executeInternal(context); assertFalse(rescore); - IndexSearcher countSearcher = shouldCollectCount ? new IndexSearcher(reader) : - getAssertingEarlyTerminationSearcher(reader, 0); + + ContextIndexSearcher countSearcher = shouldCollectCount ? newContextSearcher(reader) : + newEarlyTerminationContextSearcher(reader, 0); assertEquals(countSearcher.count(query), context.queryResult().topDocs().topDocs.totalHits.value); } @@ -196,17 +211,17 @@ public void testPostFilterDisablesCountOptimization() throws Exception { w.close(); IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = getAssertingEarlyTerminationSearcher(reader, 0); - TestSearchContext context = new TestSearchContext(null, indexShard); + TestSearchContext context = + new TestSearchContext(null, indexShard, newEarlyTerminationContextSearcher(reader, 0)); context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertEquals(1, context.queryResult().topDocs().topDocs.totalHits.value); - contextSearcher = new IndexSearcher(reader); + context.setSearcher(newContextSearcher(reader)); context.parsedPostFilter(new ParsedQuery(new MatchNoDocsQuery())); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value); reader.close(); dir.close(); @@ -226,15 +241,14 @@ public void testTerminateAfterWithFilter() throws Exception { w.close(); IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = new IndexSearcher(reader); - TestSearchContext context = new TestSearchContext(null, indexShard); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); context.terminateAfter(1); context.setSize(10); for (int i = 0; i < 10; i++) { context.parsedPostFilter(new ParsedQuery(new TermQuery(new Term("foo", Integer.toString(i))))); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertEquals(1, context.queryResult().topDocs().topDocs.totalHits.value); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); } @@ -253,27 +267,22 @@ public void testMinScoreDisablesCountOptimization() throws Exception { w.close(); IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = getAssertingEarlyTerminationSearcher(reader, 0); - TestSearchContext context = new TestSearchContext(null, indexShard); + TestSearchContext context = + new TestSearchContext(null, indexShard, newEarlyTerminationContextSearcher(reader, 0)); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); context.setSize(0); context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertEquals(1, context.queryResult().topDocs().topDocs.totalHits.value); - contextSearcher = new IndexSearcher(reader); context.minimumScore(100); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertEquals(0, context.queryResult().topDocs().topDocs.totalHits.value); reader.close(); dir.close(); } public void testQueryCapturesThreadPoolStats() throws Exception { - TestSearchContext context = new TestSearchContext(null, indexShard); - context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); - context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); - Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); @@ -283,9 +292,11 @@ public void testQueryCapturesThreadPoolStats() throws Exception { } w.close(); IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = new IndexSearcher(reader); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); + context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); + context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); QuerySearchResult results = context.queryResult(); assertThat(results.serviceTimeEWMA(), greaterThanOrEqualTo(0L)); assertThat(results.nodeQueueSize(), greaterThanOrEqualTo(0)); @@ -305,8 +316,7 @@ public void testInOrderScrollOptimization() throws Exception { } w.close(); IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = new IndexSearcher(reader); - TestSearchContext context = new TestSearchContext(null, indexShard); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); ScrollContext scrollContext = new ScrollContext(); scrollContext.lastEmittedDoc = null; @@ -317,14 +327,14 @@ public void testInOrderScrollOptimization() throws Exception { int size = randomIntBetween(2, 5); context.setSize(size); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs)); assertNull(context.queryResult().terminatedEarly()); assertThat(context.terminateAfter(), equalTo(0)); assertThat(context.queryResult().getTotalHits().value, equalTo((long) numDocs)); - contextSearcher = getAssertingEarlyTerminationSearcher(reader, size); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + context.setSearcher(newEarlyTerminationContextSearcher(reader, size)); + QueryPhase.executeInternal(context); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs)); assertThat(context.terminateAfter(), equalTo(size)); assertThat(context.queryResult().getTotalHits().value, equalTo((long) numDocs)); @@ -350,19 +360,17 @@ public void testTerminateAfterEarlyTermination() throws Exception { w.addDocument(doc); } w.close(); - TestSearchContext context = new TestSearchContext(null, indexShard); + final IndexReader reader = DirectoryReader.open(dir); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); - final IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = new IndexSearcher(reader); - context.terminateAfter(numDocs); { context.setSize(10); TotalHitCountCollector collector = new TotalHitCountCollector(); context.queryCollectors().put(TotalHitCountCollector.class, collector); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertFalse(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(10)); @@ -372,13 +380,13 @@ public void testTerminateAfterEarlyTermination() throws Exception { context.terminateAfter(1); { context.setSize(1); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); context.setSize(0); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0)); @@ -386,7 +394,7 @@ public void testTerminateAfterEarlyTermination() throws Exception { { context.setSize(1); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -398,14 +406,14 @@ public void testTerminateAfterEarlyTermination() throws Exception { .add(new TermQuery(new Term("foo", "baz")), Occur.SHOULD) .build(); context.parsedQuery(new ParsedQuery(bq)); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); context.setSize(0); context.parsedQuery(new ParsedQuery(bq)); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0)); @@ -414,7 +422,7 @@ public void testTerminateAfterEarlyTermination() throws Exception { context.setSize(1); TotalHitCountCollector collector = new TotalHitCountCollector(); context.queryCollectors().put(TotalHitCountCollector.class, collector); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -425,7 +433,7 @@ public void testTerminateAfterEarlyTermination() throws Exception { context.setSize(0); TotalHitCountCollector collector = new TotalHitCountCollector(); context.queryCollectors().put(TotalHitCountCollector.class, collector); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertTrue(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(0)); @@ -455,15 +463,15 @@ public void testIndexSortingEarlyTermination() throws Exception { } w.close(); - TestSearchContext context = new TestSearchContext(null, indexShard); + final IndexReader reader = DirectoryReader.open(dir); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); context.setSize(1); context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); context.sort(new SortAndFormats(sort, new DocValueFormat[] {DocValueFormat.RAW})); - final IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = new IndexSearcher(reader); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + + QueryPhase.executeInternal(context); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class)); @@ -472,7 +480,7 @@ public void testIndexSortingEarlyTermination() throws Exception { { context.parsedPostFilter(new ParsedQuery(new MinDocQuery(1))); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo(numDocs - 1L)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -482,7 +490,7 @@ public void testIndexSortingEarlyTermination() throws Exception { final TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector(); context.queryCollectors().put(TotalHitCountCollector.class, totalHitCountCollector); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); @@ -493,15 +501,15 @@ public void testIndexSortingEarlyTermination() throws Exception { } { - contextSearcher = getAssertingEarlyTerminationSearcher(reader, 1); + context.setSearcher(newEarlyTerminationContextSearcher(reader, 1)); context.trackTotalHitsUpTo(SearchContext.TRACK_TOTAL_HITS_DISABLED); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class)); assertThat(fieldDoc.fields[0], anyOf(equalTo(1), equalTo(2))); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); assertThat(context.queryResult().topDocs().topDocs.scoreDocs[0], instanceOf(FieldDoc.class)); @@ -537,8 +545,7 @@ public void testIndexSortScrollOptimization() throws Exception { // search sort is a prefix of the index sort searchSortAndFormats.add(new SortAndFormats(new Sort(indexSort.getSort()[0]), new DocValueFormat[]{DocValueFormat.RAW})); for (SortAndFormats searchSortAndFormat : searchSortAndFormats) { - IndexSearcher contextSearcher = new IndexSearcher(reader); - TestSearchContext context = new TestSearchContext(null, indexShard); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); context.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); ScrollContext scrollContext = new ScrollContext(); scrollContext.lastEmittedDoc = null; @@ -549,7 +556,7 @@ public void testIndexSortScrollOptimization() throws Exception { context.setSize(10); context.sort(searchSortAndFormat); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs)); assertNull(context.queryResult().terminatedEarly()); assertThat(context.terminateAfter(), equalTo(0)); @@ -557,8 +564,8 @@ public void testIndexSortScrollOptimization() throws Exception { int sizeMinus1 = context.queryResult().topDocs().topDocs.scoreDocs.length - 1; FieldDoc lastDoc = (FieldDoc) context.queryResult().topDocs().topDocs.scoreDocs[sizeMinus1]; - contextSearcher = getAssertingEarlyTerminationSearcher(reader, 10); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + context.setSearcher(newEarlyTerminationContextSearcher(reader, 10)); + QueryPhase.executeInternal(context); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.totalHits.value, equalTo((long) numDocs)); assertThat(context.terminateAfter(), equalTo(0)); @@ -579,7 +586,6 @@ public void testIndexSortScrollOptimization() throws Exception { dir.close(); } - public void testDisableTopScoreCollection() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer()); @@ -597,8 +603,7 @@ public void testDisableTopScoreCollection() throws Exception { w.close(); IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = new IndexSearcher(reader); - TestSearchContext context = new TestSearchContext(null, indexShard); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); context.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); Query q = new SpanNearQuery.Builder("title", true) .addClause(new SpanTermQuery(new Term("title", "foo"))) @@ -608,21 +613,19 @@ public void testDisableTopScoreCollection() throws Exception { context.parsedQuery(new ParsedQuery(q)); context.setSize(3); context.trackTotalHitsUpTo(3); - - TopDocsCollectorContext topDocsContext = - TopDocsCollectorContext.createTopDocsCollectorContext(context, reader, false); + TopDocsCollectorContext topDocsContext = TopDocsCollectorContext.createTopDocsCollectorContext(context, false); assertEquals(topDocsContext.create(null).scoreMode(), org.apache.lucene.search.ScoreMode.COMPLETE); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertEquals(5, context.queryResult().topDocs().topDocs.totalHits.value); assertEquals(context.queryResult().topDocs().topDocs.totalHits.relation, TotalHits.Relation.EQUAL_TO); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(3)); + context.sort(new SortAndFormats(new Sort(new SortField("other", SortField.Type.INT)), new DocValueFormat[] { DocValueFormat.RAW })); - topDocsContext = - TopDocsCollectorContext.createTopDocsCollectorContext(context, reader, false); + topDocsContext = TopDocsCollectorContext.createTopDocsCollectorContext(context, false); assertEquals(topDocsContext.create(null).scoreMode(), org.apache.lucene.search.ScoreMode.COMPLETE_NO_SCORES); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertEquals(5, context.queryResult().topDocs().topDocs.totalHits.value); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(3)); assertEquals(context.queryResult().topDocs().topDocs.totalHits.relation, TotalHits.Relation.EQUAL_TO); @@ -631,13 +634,108 @@ public void testDisableTopScoreCollection() throws Exception { dir.close(); } + public void testNumericLongOrDateSortOptimization() throws Exception { + final String fieldNameLong = "long-field"; + final String fieldNameDate = "date-field"; + MappedFieldType fieldTypeLong = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + MappedFieldType fieldTypeDate = new DateFieldMapper.Builder(fieldNameDate).fieldType(); + MapperService mapperService = mock(MapperService.class); + when(mapperService.fullName(fieldNameLong)).thenReturn(fieldTypeLong); + when(mapperService.fullName(fieldNameDate)).thenReturn(fieldTypeDate); + + final int numDocs = 7000; + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(null)); + for (int i = 1; i <= numDocs; ++i) { + Document doc = new Document(); + long longValue = randomLongBetween(-10000000L, 10000000L); + doc.add(new LongPoint(fieldNameLong, longValue)); + doc.add(new NumericDocValuesField(fieldNameLong, longValue)); + longValue = randomLongBetween(0, 3000000000000L); + doc.add(new LongPoint(fieldNameDate, longValue)); + doc.add(new NumericDocValuesField(fieldNameDate, longValue)); + writer.addDocument(doc); + if (i % 3500 == 0) writer.commit(); + } + writer.close(); + final IndexReader reader = DirectoryReader.open(dir); + + TestSearchContext searchContext = + spy(new TestSearchContext(null, indexShard, newOptimizedContextSearcher(reader, 0))); + when(searchContext.mapperService()).thenReturn(mapperService); + + // 1. Test a sort on long field + final SortField sortFieldLong = new SortField(fieldNameLong, SortField.Type.LONG); + sortFieldLong.setMissingValue(Long.MAX_VALUE); + final Sort longSort = new Sort(sortFieldLong); + SortAndFormats sortAndFormats = new SortAndFormats(longSort, new DocValueFormat[]{DocValueFormat.RAW}); + searchContext.sort(sortAndFormats); + searchContext.parsedQuery(new ParsedQuery(new MatchAllDocsQuery())); + searchContext.setTask(new SearchTask(123L, "", "", "", null, Collections.emptyMap())); + searchContext.setSize(10); + QueryPhase.executeInternal(searchContext); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, false); + + // 2. Test a sort on long field + date field + final SortField sortFieldDate = new SortField(fieldNameDate, SortField.Type.LONG); + DocValueFormat dateFormat = fieldTypeDate.docValueFormat(null, null); + final Sort longDateSort = new Sort(sortFieldLong, sortFieldDate); + sortAndFormats = new SortAndFormats(longDateSort, new DocValueFormat[]{DocValueFormat.RAW, dateFormat}); + searchContext.sort(sortAndFormats); + QueryPhase.executeInternal(searchContext); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, true); + + // 3. Test a sort on date field + sortFieldDate.setMissingValue(Long.MAX_VALUE); + final Sort dateSort = new Sort(sortFieldDate); + sortAndFormats = new SortAndFormats(dateSort, new DocValueFormat[]{dateFormat}); + searchContext.sort(sortAndFormats); + QueryPhase.executeInternal(searchContext); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, false); + + // 4. Test a sort on date field + long field + final Sort dateLongSort = new Sort(sortFieldDate, sortFieldLong); + sortAndFormats = new SortAndFormats(dateLongSort, new DocValueFormat[]{dateFormat, DocValueFormat.RAW}); + searchContext.sort(sortAndFormats); + QueryPhase.executeInternal(searchContext); + assertSortResults(searchContext.queryResult().topDocs().topDocs, (long) numDocs, true); + reader.close(); + dir.close(); + } + + public void testIndexHasDuplicateData() throws IOException { + int docsCount = 7000; + int duplIndex = docsCount * 7 / 10; + int duplIndex2 = docsCount * 3 / 10; + long duplicateValue = randomLongBetween(-10000000L, 10000000L); + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(null)); + for (int docId = 0; docId < docsCount; docId++) { + Document doc = new Document(); + long rndValue = randomLongBetween(-10000000L, 10000000L); + long value = (docId < duplIndex) ? duplicateValue : rndValue; + long value2 = (docId < duplIndex2) ? duplicateValue : rndValue; + doc.add(new LongPoint("duplicateField", value)); + doc.add(new LongPoint("notDuplicateField", value2)); + writer.addDocument(doc); + } + writer.close(); + final IndexReader reader = DirectoryReader.open(dir); + boolean hasDuplicateData = indexFieldHasDuplicateData(reader, "duplicateField"); + boolean hasDuplicateData2 = indexFieldHasDuplicateData(reader, "notDuplicateField"); + reader.close(); + dir.close(); + assertTrue(hasDuplicateData); + assertFalse(hasDuplicateData2); + } + public void testMaxScoreQueryVisitor() { BitSetProducer producer = context -> new FixedBitSet(1); Query query = new ESToParentBlockJoinQuery(new MatchAllDocsQuery(), producer, ScoreMode.Avg, "nested"); - assertTrue(TopDocsCollectorContext.hasInfMaxScore(query)); + assertTrue(hasInfMaxScore(query)); query = new ESToParentBlockJoinQuery(new MatchAllDocsQuery(), producer, ScoreMode.None, "nested"); - assertFalse(TopDocsCollectorContext.hasInfMaxScore(query)); + assertFalse(hasInfMaxScore(query)); for (Occur occur : Occur.values()) { @@ -645,9 +743,9 @@ public void testMaxScoreQueryVisitor() { .add(new ESToParentBlockJoinQuery(new MatchAllDocsQuery(), producer, ScoreMode.Avg, "nested"), occur) .build(); if (occur == Occur.MUST) { - assertTrue(TopDocsCollectorContext.hasInfMaxScore(query)); + assertTrue(hasInfMaxScore(query)); } else { - assertFalse(TopDocsCollectorContext.hasInfMaxScore(query)); + assertFalse(hasInfMaxScore(query)); } query = new BooleanQuery.Builder() @@ -656,9 +754,9 @@ public void testMaxScoreQueryVisitor() { .build(), occur) .build(); if (occur == Occur.MUST) { - assertTrue(TopDocsCollectorContext.hasInfMaxScore(query)); + assertTrue(hasInfMaxScore(query)); } else { - assertFalse(TopDocsCollectorContext.hasInfMaxScore(query)); + assertFalse(hasInfMaxScore(query)); } query = new BooleanQuery.Builder() @@ -666,7 +764,7 @@ public void testMaxScoreQueryVisitor() { .add(new ESToParentBlockJoinQuery(new MatchAllDocsQuery(), producer, ScoreMode.Avg, "nested"), occur) .build(), Occur.FILTER) .build(); - assertFalse(TopDocsCollectorContext.hasInfMaxScore(query)); + assertFalse(hasInfMaxScore(query)); query = new BooleanQuery.Builder() .add(new BooleanQuery.Builder() @@ -675,13 +773,33 @@ public void testMaxScoreQueryVisitor() { .build(), occur) .build(); if (occur == Occur.MUST) { - assertTrue(TopDocsCollectorContext.hasInfMaxScore(query)); + assertTrue(hasInfMaxScore(query)); } else { - assertFalse(TopDocsCollectorContext.hasInfMaxScore(query)); + assertFalse(hasInfMaxScore(query)); } } } + // assert score docs are in order and their number is as expected + private void assertSortResults(TopDocs topDocs, long expectedNumDocs, boolean isDoubleSort) { + assertEquals(topDocs.totalHits.value, expectedNumDocs); + long cur1, cur2; + long prev1 = Long.MIN_VALUE; + long prev2 = Long.MIN_VALUE; + for (ScoreDoc scoreDoc : topDocs.scoreDocs) { + cur1 = (long) ((FieldDoc) scoreDoc).fields[0]; + assertThat(cur1, greaterThanOrEqualTo(prev1)); // test that docs are properly sorted on the first sort + if (isDoubleSort) { + cur2 = (long) ((FieldDoc) scoreDoc).fields[1]; + if (cur1 == prev1) { + assertThat(cur2, greaterThanOrEqualTo(prev2)); // test that docs are properly sorted on the secondary sort + } + prev2 = cur2; + } + prev1 = cur1; + } + } + public void testMinScore() throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); @@ -695,8 +813,7 @@ public void testMinScore() throws Exception { w.close(); IndexReader reader = DirectoryReader.open(dir); - IndexSearcher contextSearcher = new IndexSearcher(reader); - TestSearchContext context = new TestSearchContext(null, indexShard); + TestSearchContext context = new TestSearchContext(null, indexShard, newContextSearcher(reader)); context.parsedQuery(new ParsedQuery( new BooleanQuery.Builder() .add(new TermQuery(new Term("foo", "bar")), Occur.MUST) @@ -708,23 +825,61 @@ public void testMinScore() throws Exception { context.setSize(1); context.trackTotalHitsUpTo(5); - QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); + QueryPhase.executeInternal(context); assertEquals(10, context.queryResult().topDocs().topDocs.totalHits.value); reader.close(); dir.close(); + } - private static IndexSearcher getAssertingEarlyTerminationSearcher(IndexReader reader, int size) { - return new IndexSearcher(reader) { + private static ContextIndexSearcher newContextSearcher(IndexReader reader) { + return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()); + } + + private static ContextIndexSearcher newEarlyTerminationContextSearcher(IndexReader reader, int size) { + return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()) { + @Override - protected void search(List leaves, Weight weight, Collector collector) throws IOException { + public void search(List leaves, Weight weight, Collector collector) throws IOException { final Collector in = new AssertingEarlyTerminationFilterCollector(collector, size); super.search(leaves, weight, in); } }; } + // used to check that numeric long or date sort optimization was run + private static ContextIndexSearcher newOptimizedContextSearcher(IndexReader reader, int queryType) { + return new ContextIndexSearcher(reader, IndexSearcher.getDefaultSimilarity(), + IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy()) { + + @Override + public void search(List leaves, Weight weight, CollectorManager manager, + QuerySearchResult result, DocValueFormat[] formats, TotalHits totalHits) throws IOException { + final Query query = weight.getQuery(); + assertTrue(query instanceof BooleanQuery); + List clauses = ((BooleanQuery) query).clauses(); + assertTrue(clauses.size() == 2); + assertTrue(clauses.get(0).getOccur() == Occur.FILTER); + assertTrue(clauses.get(1).getOccur() == Occur.SHOULD); + if (queryType == 0) { + assertTrue (clauses.get(1).getQuery().getClass() == + LongPoint.newDistanceFeatureQuery("random_field", 1, 1, 1).getClass() + ); + } + if (queryType == 1) assertTrue(clauses.get(1).getQuery() instanceof DocValuesFieldExistsQuery); + super.search(leaves, weight, manager, result, formats, totalHits); + } + + @Override + public void search(List leaves, Weight weight, Collector collector) { + assert(false); // should not be there, expected to search with CollectorManager + } + }; + } + private static class AssertingEarlyTerminationFilterCollector extends FilterCollector { private final int size; diff --git a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java index 53948fdca7e42..fe24db5161d1c 100644 --- a/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java +++ b/server/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.query.WrapperQueryBuilder; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.index.search.MatchQuery; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.TermsLookup; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; @@ -1723,18 +1724,28 @@ public void testFieldAliasesForMetaFields() throws Exception { .setRouting("custom") .setSource("field", "value"); indexRandom(true, false, indexRequest); + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true)) + .get(); + try { + SearchResponse searchResponse = client().prepareSearch() + .setQuery(termQuery("routing-alias", "custom")) + .addDocValueField("id-alias") + .get(); + assertHitCount(searchResponse, 1L); + + SearchHit hit = searchResponse.getHits().getAt(0); + assertEquals(2, hit.getFields().size()); + assertTrue(hit.getFields().containsKey("id-alias")); + + DocumentField field = hit.getFields().get("id-alias"); + assertThat(field.getValue().toString(), equalTo("1")); + } finally { + // unset cluster setting + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())) + .get(); + } - SearchResponse searchResponse = client().prepareSearch() - .setQuery(termQuery("routing-alias", "custom")) - .addDocValueField("id-alias") - .get(); - assertHitCount(searchResponse, 1L); - - SearchHit hit = searchResponse.getHits().getAt(0); - assertEquals(2, hit.getFields().size()); - assertTrue(hit.getFields().containsKey("id-alias")); - - DocumentField field = hit.getFields().get("id-alias"); - assertThat(field.getValue().toString(), equalTo("1")); } } diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java index 1157cfe11f9c5..fb4e3c8e828e6 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -24,6 +24,7 @@ import org.apache.lucene.util.UnicodeUtil; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; @@ -36,6 +37,7 @@ import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.MockScriptPlugin; @@ -80,8 +82,10 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -1371,31 +1375,42 @@ public void testSortOnRareField() throws IOException { } public void testSortMetaField() throws Exception { - createIndex("test"); - ensureGreen(); - final int numDocs = randomIntBetween(10, 20); - IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; - for (int i = 0; i < numDocs; ++i) { - indexReqs[i] = client().prepareIndex("test").setId(Integer.toString(i)) + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().put(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey(), true)) + .get(); + try { + createIndex("test"); + ensureGreen(); + final int numDocs = randomIntBetween(10, 20); + IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; + for (int i = 0; i < numDocs; ++i) { + indexReqs[i] = client().prepareIndex("test").setId(Integer.toString(i)) .setSource(); - } - indexRandom(true, indexReqs); + } + indexRandom(true, indexReqs); - SortOrder order = randomFrom(SortOrder.values()); - SearchResponse searchResponse = client().prepareSearch() + SortOrder order = randomFrom(SortOrder.values()); + SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .setSize(randomIntBetween(1, numDocs + 5)) .addSort("_id", order) .get(); - assertNoFailures(searchResponse); - SearchHit[] hits = searchResponse.getHits().getHits(); - BytesRef previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; - for (int i = 0; i < hits.length; ++i) { - String idString = hits[i].getId(); - final BytesRef id = new BytesRef(idString); - assertEquals(idString, hits[i].getSortValues()[0]); - assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id)); - previous = id; + assertNoFailures(searchResponse); + SearchHit[] hits = searchResponse.getHits().getHits(); + BytesRef previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; + for (int i = 0; i < hits.length; ++i) { + String idString = hits[i].getId(); + final BytesRef id = new BytesRef(idString); + assertEquals(idString, hits[i].getSortValues()[0]); + assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id)); + previous = id; + } + // assertWarnings(ID_FIELD_DATA_DEPRECATION_MESSAGE); + } finally { + // unset cluster setting + client().admin().cluster().prepareUpdateSettings() + .setTransientSettings(Settings.builder().putNull(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())) + .get(); } } @@ -1841,4 +1856,50 @@ public void testCastNumericTypeExceptions() throws Exception { } } } + + public void testLongSortOptimizationCorrectResults() { + assertAcked(prepareCreate("test1") + .setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)) + .addMapping("_doc", "long_field", "type=long").get()); + + BulkRequestBuilder bulkBuilder = client().prepareBulk(); + for (int i = 1; i <= 7000; i++) { + if (i % 3500 == 0) { + bulkBuilder.get(); + bulkBuilder = client().prepareBulk(); + } + String source = "{\"long_field\":" + randomLong() + "}"; + bulkBuilder.add(client().prepareIndex("test1").setId(Integer.toString(i)).setSource(source, XContentType.JSON)); + } + refresh(); + + //*** 1. sort DESC on long_field + SearchResponse searchResponse = client().prepareSearch() + .addSort(new FieldSortBuilder("long_field").order(SortOrder.DESC)) + .setSize(10).get(); + assertSearchResponse(searchResponse); + long previousLong = Long.MAX_VALUE; + for (int i = 0; i < searchResponse.getHits().getHits().length; i++) { + // check the correct sort order + SearchHit hit = searchResponse.getHits().getHits()[i]; + long currentLong = (long) hit.getSortValues()[0]; + assertThat("sort order is incorrect", currentLong, lessThanOrEqualTo(previousLong)); + previousLong = currentLong; + } + + //*** 2. sort ASC on long_field + searchResponse = client().prepareSearch() + .addSort(new FieldSortBuilder("long_field").order(SortOrder.ASC)) + .setSize(10).get(); + assertSearchResponse(searchResponse); + previousLong = Long.MIN_VALUE; + for (int i = 0; i < searchResponse.getHits().getHits().length; i++) { + // check the correct sort order + SearchHit hit = searchResponse.getHits().getHits()[i]; + long currentLong = (long) hit.getSortValues()[0]; + assertThat("sort order is incorrect", currentLong, greaterThanOrEqualTo(previousLong)); + previousLong = currentLong; + } + } + } diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java index bb62f7692d185..7dcaaf16f4a37 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/FakeOAuth2HttpHandler.java @@ -30,12 +30,20 @@ @SuppressForbidden(reason = "Uses a HttpServer to emulate a fake OAuth2 authentication service") public class FakeOAuth2HttpHandler implements HttpHandler { + private static final byte[] BUFFER = new byte[1024]; + @Override public void handle(final HttpExchange exchange) throws IOException { - byte[] response = ("{\"access_token\":\"foo\",\"token_type\":\"Bearer\",\"expires_in\":3600}").getBytes(UTF_8); - exchange.getResponseHeaders().add("Content-Type", "application/json"); - exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); - exchange.getResponseBody().write(response); - exchange.close(); + try { + byte[] response = ("{\"access_token\":\"foo\",\"token_type\":\"Bearer\",\"expires_in\":3600}").getBytes(UTF_8); + exchange.getResponseHeaders().add("Content-Type", "application/json"); + exchange.sendResponseHeaders(RestStatus.OK.getStatus(), response.length); + exchange.getResponseBody().write(response); + while (exchange.getRequestBody().read(BUFFER) >= 0) ; + } finally { + int read = exchange.getRequestBody().read(); + assert read == -1 : "Request body should have been fully read here but saw [" + read + "]"; + exchange.close(); + } } } diff --git a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java index 6c007eb3734de..5a359ad2c7cc0 100644 --- a/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java +++ b/test/fixtures/gcs-fixture/src/main/java/fixture/gcs/GoogleCloudStorageHttpHandler.java @@ -81,6 +81,8 @@ public void handle(final HttpExchange exchange) throws IOException { assert read == -1 : "Request body should have been empty but saw [" + read + "]"; } try { + // Request body is closed in the finally block + final InputStream wrappedRequest = Streams.noCloseStream(exchange.getRequestBody()); if (Regex.simpleMatch("GET /storage/v1/b/" + bucket + "/o*", request)) { // List Objects https://cloud.google.com/storage/docs/json_api/v1/objects/list final Map params = new HashMap<>(); @@ -159,7 +161,7 @@ public void handle(final HttpExchange exchange) throws IOException { // Batch https://cloud.google.com/storage/docs/json_api/v1/how-tos/batch final String uri = "/storage/v1/b/" + bucket + "/o/"; final StringBuilder batch = new StringBuilder(); - for (String line : Streams.readAllLines(new BufferedInputStream(exchange.getRequestBody()))) { + for (String line : Streams.readAllLines(new BufferedInputStream(wrappedRequest))) { if (line.length() == 0 || line.startsWith("--") || line.toLowerCase(Locale.ROOT).startsWith("content")) { batch.append(line).append('\n'); } else if (line.startsWith("DELETE")) { @@ -179,7 +181,7 @@ public void handle(final HttpExchange exchange) throws IOException { } else if (Regex.simpleMatch("POST /upload/storage/v1/b/" + bucket + "/*uploadType=multipart*", request)) { // Multipart upload - Optional> content = parseMultipartRequestBody(exchange.getRequestBody()); + Optional> content = parseMultipartRequestBody(wrappedRequest); if (content.isPresent()) { blobs.put(content.get().v1(), content.get().v2()); @@ -198,7 +200,7 @@ public void handle(final HttpExchange exchange) throws IOException { final String blobName = params.get("name"); blobs.put(blobName, BytesArray.EMPTY); - byte[] response = Streams.readFully(exchange.getRequestBody()).utf8ToString().getBytes(UTF_8); + byte[] response = Streams.readFully(wrappedRequest).utf8ToString().getBytes(UTF_8); exchange.getResponseHeaders().add("Content-Type", "application/json"); exchange.getResponseHeaders().add("Location", httpServerUrl(exchange) + "/upload/storage/v1/b/" + bucket + "/o?" + "uploadType=resumable" @@ -224,7 +226,7 @@ public void handle(final HttpExchange exchange) throws IOException { final int end = getContentRangeEnd(range); final ByteArrayOutputStream out = new ByteArrayOutputStream(); - long bytesRead = Streams.copy(exchange.getRequestBody(), out); + long bytesRead = Streams.copy(wrappedRequest, out); int length = Math.max(end + 1, limit != null ? limit : 0); if ((int) bytesRead > length) { throw new AssertionError("Requesting more bytes than available for blob"); @@ -249,6 +251,8 @@ public void handle(final HttpExchange exchange) throws IOException { exchange.sendResponseHeaders(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), -1); } } finally { + int read = exchange.getRequestBody().read(); + assert read == -1 : "Request body should have been fully read here but saw [" + read + "]"; exchange.close(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java index 05421edc58f08..7711c4c5bd365 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java +++ b/test/framework/src/main/java/org/elasticsearch/index/MapperTestUtils.java @@ -66,6 +66,6 @@ public static MapperService newMapperService(NamedXContentRegistry xContentRegis xContentRegistry, similarityService, mapperRegistry, - () -> null); + () -> null, () -> false); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java index 7504c97de934b..2a676bb7b7750 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/EngineTestCase.java @@ -1010,7 +1010,7 @@ public static List getDocIds(Engine engine, boolean refresh if (refresh) { engine.refresh("test_get_doc_ids"); } - try (Engine.Searcher searcher = engine.acquireSearcher("test_get_doc_ids")) { + try (Engine.Searcher searcher = engine.acquireSearcher("test_get_doc_ids", Engine.SearcherScope.INTERNAL)) { List docs = new ArrayList<>(); for (LeafReaderContext leafContext : searcher.getIndexReader().leaves()) { LeafReader reader = leafContext.reader(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 144efde14ba38..5adc5f86699f9 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -65,7 +65,7 @@ public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings ind SimilarityService similarityService = new SimilarityService(indexSettings, null, emptyMap()); MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry(); mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, - () -> null); + () -> null, () -> false); } private DocumentMapperForType docMapper(String type) { diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java index 21071f7cb5005..2b273c1e6a784 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreContainerTestCase.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.blobstore.BlobStore; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; import org.elasticsearch.test.ESTestCase; import java.io.IOException; @@ -36,8 +37,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; -import static org.elasticsearch.repositories.ESBlobStoreTestCase.writeRandomBlob; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.notNullValue; @@ -191,5 +190,51 @@ protected void writeBlob(final BlobContainer container, final String blobName, f } } + public void testContainerCreationAndDeletion() throws IOException { + try(BlobStore store = newBlobStore()) { + final BlobContainer containerFoo = store.blobContainer(new BlobPath().add("foo")); + final BlobContainer containerBar = store.blobContainer(new BlobPath().add("bar")); + byte[] data1 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); + byte[] data2 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); + writeBlob(containerFoo, "test", new BytesArray(data1)); + writeBlob(containerBar, "test", new BytesArray(data2)); + + assertArrayEquals(readBlobFully(containerFoo, "test", data1.length), data1); + assertArrayEquals(readBlobFully(containerBar, "test", data2.length), data2); + + assertTrue(BlobStoreTestUtil.blobExists(containerFoo, "test")); + assertTrue(BlobStoreTestUtil.blobExists(containerBar, "test")); + } + } + + public static byte[] writeRandomBlob(BlobContainer container, String name, int length) throws IOException { + byte[] data = randomBytes(length); + writeBlob(container, name, new BytesArray(data)); + return data; + } + + public static byte[] readBlobFully(BlobContainer container, String name, int length) throws IOException { + byte[] data = new byte[length]; + try (InputStream inputStream = container.readBlob(name)) { + assertThat(inputStream.read(data), equalTo(length)); + assertThat(inputStream.read(), equalTo(-1)); + } + return data; + } + + public static byte[] randomBytes(int length) { + byte[] data = new byte[length]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) randomInt(); + } + return data; + } + + protected static void writeBlob(BlobContainer container, String blobName, BytesArray bytesArray) throws IOException { + try (InputStream stream = bytesArray.streamInput()) { + container.writeBlob(blobName, stream, bytesArray.length(), true); + } + } + protected abstract BlobStore newBlobStore() throws IOException; } diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java deleted file mode 100644 index fe6f059fd38e3..0000000000000 --- a/test/framework/src/main/java/org/elasticsearch/repositories/ESBlobStoreTestCase.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.repositories; - -import org.elasticsearch.common.blobstore.BlobContainer; -import org.elasticsearch.common.blobstore.BlobPath; -import org.elasticsearch.common.blobstore.BlobStore; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.repositories.blobstore.BlobStoreTestUtil; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; -import java.io.InputStream; - -import static org.hamcrest.CoreMatchers.equalTo; - -/** - * Generic test case for blob store implementation. - * These tests check basic blob store functionality. - */ -public abstract class ESBlobStoreTestCase extends ESTestCase { - - public void testContainerCreationAndDeletion() throws IOException { - try(BlobStore store = newBlobStore()) { - final BlobContainer containerFoo = store.blobContainer(new BlobPath().add("foo")); - final BlobContainer containerBar = store.blobContainer(new BlobPath().add("bar")); - byte[] data1 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); - byte[] data2 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); - writeBlob(containerFoo, "test", new BytesArray(data1)); - writeBlob(containerBar, "test", new BytesArray(data2)); - - assertArrayEquals(readBlobFully(containerFoo, "test", data1.length), data1); - assertArrayEquals(readBlobFully(containerBar, "test", data2.length), data2); - - assertTrue(BlobStoreTestUtil.blobExists(containerFoo, "test")); - assertTrue(BlobStoreTestUtil.blobExists(containerBar, "test")); - } - } - - public static byte[] writeRandomBlob(BlobContainer container, String name, int length) throws IOException { - byte[] data = randomBytes(length); - writeBlob(container, name, new BytesArray(data)); - return data; - } - - public static byte[] readBlobFully(BlobContainer container, String name, int length) throws IOException { - byte[] data = new byte[length]; - try (InputStream inputStream = container.readBlob(name)) { - assertThat(inputStream.read(data), equalTo(length)); - assertThat(inputStream.read(), equalTo(-1)); - } - return data; - } - - public static byte[] randomBytes(int length) { - byte[] data = new byte[length]; - for (int i = 0; i < data.length; i++) { - data[i] = (byte) randomInt(); - } - return data; - } - - protected static void writeBlob(BlobContainer container, String blobName, BytesArray bytesArray) throws IOException { - try (InputStream stream = bytesArray.streamInput()) { - container.writeBlob(blobName, stream, bytesArray.length(), true); - } - } - - protected abstract BlobStore newBlobStore() throws IOException; -} diff --git a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java index 040961ed52b1d..03f89125ad979 100644 --- a/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/repositories/blobstore/ESMockAPIBasedRepositoryIntegTestCase.java @@ -190,16 +190,26 @@ protected ErroneousHttpHandler(final HttpHandler delegate, final int maxErrorsPe @Override public void handle(final HttpExchange exchange) throws IOException { - final String requestId = requestUniqueId(exchange); - assert Strings.hasText(requestId); - - final boolean canFailRequest = canFailRequest(exchange); - final int count = requests.computeIfAbsent(requestId, req -> new AtomicInteger(0)).incrementAndGet(); - if (count >= maxErrorsPerRequest || canFailRequest == false) { - requests.remove(requestId); - delegate.handle(exchange); - } else { - handleAsError(exchange); + try { + final String requestId = requestUniqueId(exchange); + assert Strings.hasText(requestId); + + final boolean canFailRequest = canFailRequest(exchange); + final int count = requests.computeIfAbsent(requestId, req -> new AtomicInteger(0)).incrementAndGet(); + if (count >= maxErrorsPerRequest || canFailRequest == false) { + requests.remove(requestId); + delegate.handle(exchange); + } else { + handleAsError(exchange); + } + } finally { + try { + int read = exchange.getRequestBody().read(); + assert read == -1 : "Request body should have been fully read here but saw [" + read + "]"; + } catch (IOException e) { + // ignored, stream is assumed to have been closed by previous handler + } + exchange.close(); } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index e726b4dc15094..07420ce2d3f7b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -356,7 +356,7 @@ private static class ServiceHolder implements Closeable { similarityService = new SimilarityService(idxSettings, null, Collections.emptyMap()); MapperRegistry mapperRegistry = indicesModule.getMapperRegistry(); mapperService = new MapperService(idxSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, - () -> createShardContext(null)); + () -> createShardContext(null), () -> false); IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(nodeSettings, new IndexFieldDataCache.Listener() { }); indexFieldDataService = new IndexFieldDataService(idxSettings, indicesFieldDataCache, diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 83da817f64bf2..e6a0821b19115 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -106,11 +106,20 @@ public TestSearchContext(QueryShardContext queryShardContext) { } public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard) { + this(queryShardContext, indexShard, null); + } + + public TestSearchContext(QueryShardContext queryShardContext, IndexShard indexShard, ContextIndexSearcher searcher) { this.bigArrays = null; this.indexService = null; this.fixedBitSetFilterCache = null; this.indexShard = indexShard; this.queryShardContext = queryShardContext; + this.searcher = searcher; + } + + public void setSearcher(ContextIndexSearcher searcher) { + this.searcher = searcher; } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index a66fdf70b617a..febe9a8eb42a9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -116,7 +116,7 @@ public final class Messages { public static final String JOB_AUDIT_DELETING_FAILED = "Error deleting job: {0}"; public static final String JOB_AUDIT_DELETED = "Job deleted"; public static final String JOB_AUDIT_KILLING = "Killing job"; - public static final String JOB_AUDIT_OLD_RESULTS_DELETED = "Deleted results prior to {1}"; + public static final String JOB_AUDIT_OLD_RESULTS_DELETED = "Deleted results prior to {0}"; public static final String JOB_AUDIT_REVERTED = "Job model snapshot reverted to ''{0}''"; public static final String JOB_AUDIT_SNAPSHOT_DELETED = "Model snapshot [{0}] with description ''{1}'' deleted"; public static final String JOB_AUDIT_FILTER_UPDATED_ON_PROCESS = "Updated filter [{0}] in running process"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java index 17c3e05a772ce..f0a34e94655da 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java @@ -207,6 +207,19 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws return builder.endObject(); } + @Override + public String toString() { + StringBuilder builder = new StringBuilder("Authentication[") + .append(user) + .append(",type=").append(type) + .append(",by=").append(authenticatedBy); + if (lookedUpBy != null) { + builder.append(",lookup=").append(lookedUpBy); + } + builder.append("]"); + return builder.toString(); + } + public static class RealmRef { private final String nodeName; @@ -262,6 +275,11 @@ public int hashCode() { result = 31 * result + type.hashCode(); return result; } + + @Override + public String toString() { + return "{Realm[" + type + "." + name + "] on Node[" + nodeName + "]}"; + } } public enum AuthenticationType { diff --git a/x-pack/plugin/enrich/build.gradle b/x-pack/plugin/enrich/build.gradle index d216a8bbacabe..12ca16115f8d6 100644 --- a/x-pack/plugin/enrich/build.gradle +++ b/x-pack/plugin/enrich/build.gradle @@ -13,6 +13,7 @@ dependencies { compileOnly project(path: xpackModule('core'), configuration: 'default') testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: ':modules:ingest-common') + testCompile project(path: ':modules:lang-mustache') testCompile project(path: xpackModule('monitoring'), configuration: 'testArtifacts') } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java index 6b77a096ea749..07be2f59cbda7 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/AbstractEnrichProcessor.java @@ -14,6 +14,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.script.TemplateScript; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; @@ -28,8 +29,8 @@ public abstract class AbstractEnrichProcessor extends AbstractProcessor { private final String policyName; private final BiConsumer> searchRunner; - private final String field; - private final String targetField; + private final TemplateScript.Factory field; + private final TemplateScript.Factory targetField; private final boolean ignoreMissing; private final boolean overrideEnabled; protected final String matchField; @@ -39,8 +40,8 @@ protected AbstractEnrichProcessor( String tag, Client client, String policyName, - String field, - String targetField, + TemplateScript.Factory field, + TemplateScript.Factory targetField, boolean ignoreMissing, boolean overrideEnabled, String matchField, @@ -53,8 +54,8 @@ protected AbstractEnrichProcessor( String tag, BiConsumer> searchRunner, String policyName, - String field, - String targetField, + TemplateScript.Factory field, + TemplateScript.Factory targetField, boolean ignoreMissing, boolean overrideEnabled, String matchField, @@ -77,6 +78,7 @@ protected AbstractEnrichProcessor( public void execute(IngestDocument ingestDocument, BiConsumer handler) { try { // If a document does not have the enrich key, return the unchanged document + String field = ingestDocument.renderTemplate(this.field); final Object value = ingestDocument.getFieldValue(field, Object.class, ignoreMissing); if (value == null) { handler.accept(ingestDocument, null); @@ -111,6 +113,7 @@ public void execute(IngestDocument ingestDocument, BiConsumer firstDocument = searchHits[0].getSourceAsMap(); @@ -146,11 +149,13 @@ public String getType() { } String getField() { - return field; + // used for testing only: + return field.newInstance(Map.of()).execute(); } - public String getTargetField() { - return targetField; + String getTargetField() { + // used for testing only: + return targetField.newInstance(Map.of()).execute(); } boolean isIgnoreMissing() { diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java index b3b4160958f8c..327c4f240f438 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichPlugin.java @@ -132,7 +132,7 @@ public Map getProcessors(Processor.Parameters paramet return Map.of(); } - EnrichProcessorFactory factory = new EnrichProcessorFactory(parameters.client); + EnrichProcessorFactory factory = new EnrichProcessorFactory(parameters.client, parameters.scriptService); parameters.ingestService.addIngestClusterStateListener(factory); return Map.of(EnrichProcessorFactory.TYPE, factory); } diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java index 0cd8995923628..96d9efadde750 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactory.java @@ -14,6 +14,8 @@ import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import java.util.Map; @@ -23,11 +25,13 @@ final class EnrichProcessorFactory implements Processor.Factory, Consumer processorFactories, Strin assert aliasOrIndex.getIndices().size() == 1; IndexMetaData imd = aliasOrIndex.getIndices().get(0); - String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); Map mappingAsMap = imd.mapping().sourceAsMap(); String policyType = (String) XContentMapValues.extractValue( "_meta." + EnrichPolicyRunner.ENRICH_POLICY_TYPE_FIELD_NAME, @@ -50,9 +53,10 @@ public Processor create(Map processorFactories, Strin ); String matchField = (String) XContentMapValues.extractValue("_meta." + EnrichPolicyRunner.ENRICH_MATCH_FIELD_NAME, mappingAsMap); + TemplateScript.Factory field = ConfigurationUtils.readTemplateProperty(TYPE, tag, config, "field", scriptService); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "ignore_missing", false); boolean overrideEnabled = ConfigurationUtils.readBooleanProperty(TYPE, tag, config, "override", true); - String targetField = ConfigurationUtils.readStringProperty(TYPE, tag, config, "target_field"); + TemplateScript.Factory targetField = ConfigurationUtils.readTemplateProperty(TYPE, tag, config, "target_field", scriptService); int maxMatches = ConfigurationUtils.readIntProperty(TYPE, tag, config, "max_matches", 1); if (maxMatches < 1 || maxMatches > 128) { throw ConfigurationUtils.newConfigurationException(TYPE, tag, "max_matches", "should be between 1 and 128"); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java index 73d224d74ec57..b10bafa5e959c 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/GeoMatchProcessor.java @@ -16,6 +16,7 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.index.query.GeoShapeQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.script.TemplateScript; import java.util.ArrayList; import java.util.List; @@ -29,8 +30,8 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor { String tag, Client client, String policyName, - String field, - String targetField, + TemplateScript.Factory field, + TemplateScript.Factory targetField, boolean overrideEnabled, boolean ignoreMissing, String matchField, @@ -46,8 +47,8 @@ public final class GeoMatchProcessor extends AbstractEnrichProcessor { String tag, BiConsumer> searchRunner, String policyName, - String field, - String targetField, + TemplateScript.Factory field, + TemplateScript.Factory targetField, boolean overrideEnabled, boolean ignoreMissing, String matchField, diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java index 44cef89fe9bd3..6e2967272fa26 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/MatchProcessor.java @@ -11,6 +11,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; +import org.elasticsearch.script.TemplateScript; import java.util.List; import java.util.function.BiConsumer; @@ -21,8 +22,8 @@ public class MatchProcessor extends AbstractEnrichProcessor { String tag, Client client, String policyName, - String field, - String targetField, + TemplateScript.Factory field, + TemplateScript.Factory targetField, boolean overrideEnabled, boolean ignoreMissing, String matchField, @@ -36,8 +37,8 @@ public class MatchProcessor extends AbstractEnrichProcessor { String tag, BiConsumer> searchRunner, String policyName, - String field, - String targetField, + TemplateScript.Factory field, + TemplateScript.Factory targetField, boolean overrideEnabled, boolean ignoreMissing, String matchField, diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java index 35e63855cd67d..aaa0ef2b1392a 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/BasicEnrichTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.index.reindex.ReindexPlugin; import org.elasticsearch.ingest.common.IngestCommonPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.mustache.MustachePlugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.core.enrich.action.EnrichStatsAction; @@ -49,7 +50,7 @@ public class BasicEnrichTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return List.of(LocalStateEnrich.class, ReindexPlugin.class, IngestCommonPlugin.class); + return List.of(LocalStateEnrich.class, ReindexPlugin.class, IngestCommonPlugin.class, MustachePlugin.class); } @Override @@ -297,6 +298,43 @@ public void testAsyncTaskExecute() throws Exception { } } + public void testTemplating() throws Exception { + List keys = createSourceMatchIndex(1, 1); + String policyName = "my-policy"; + EnrichPolicy enrichPolicy = new EnrichPolicy( + EnrichPolicy.MATCH_TYPE, + null, + List.of(SOURCE_INDEX_NAME), + MATCH_FIELD, + List.of(DECORATE_FIELDS) + ); + PutEnrichPolicyAction.Request request = new PutEnrichPolicyAction.Request(policyName, enrichPolicy); + client().execute(PutEnrichPolicyAction.INSTANCE, request).actionGet(); + client().execute(ExecuteEnrichPolicyAction.INSTANCE, new ExecuteEnrichPolicyAction.Request(policyName)).actionGet(); + + String pipelineName = "my-pipeline"; + String pipelineBody = "{\"processors\": [{\"enrich\": {\"policy_name\":\"" + + policyName + + "\", \"field\": \"{{indirection1}}\", \"target_field\": \"{{indirection2}}\"" + + "}}]}"; + PutPipelineRequest putPipelineRequest = new PutPipelineRequest(pipelineName, new BytesArray(pipelineBody), XContentType.JSON); + client().admin().cluster().putPipeline(putPipelineRequest).actionGet(); + + IndexRequest indexRequest = new IndexRequest("my-index").id("1") + .setPipeline(pipelineName) + .source(Map.of("indirection1", MATCH_FIELD, "indirection2", "users", MATCH_FIELD, keys.get(0))); + client().index(indexRequest).get(); + GetResponse getResponse = client().get(new GetRequest("my-index", "1")).actionGet(); + Map source = getResponse.getSourceAsMap(); + Map userEntry = (Map) source.get("users"); + assertThat(userEntry.size(), equalTo(DECORATE_FIELDS.length + 1)); + for (int j = 0; j < 3; j++) { + String field = DECORATE_FIELDS[j]; + assertThat(userEntry.get(field), equalTo(keys.get(0) + j)); + } + assertThat(keys.contains(userEntry.get(MATCH_FIELD)), is(true)); + } + private List createSourceMatchIndex(int numKeys, int numDocsPerKey) { Set keys = new HashSet<>(); for (int id = 0; id < numKeys; id++) { diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java index 465d18f96175a..306ab9a6962b1 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/EnrichProcessorFactoryTests.java @@ -12,8 +12,10 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; +import org.junit.Before; import java.io.IOException; import java.util.ArrayList; @@ -25,13 +27,21 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.Mockito.mock; public class EnrichProcessorFactoryTests extends ESTestCase { + private ScriptService scriptService; + + @Before + public void initializeScriptService() { + scriptService = mock(ScriptService.class); + } + public void testCreateProcessorInstance() throws Exception { List enrichValues = List.of("globalRank", "tldRank", "tld"); EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("source_index"), "my_key", enrichValues); - EnrichProcessorFactory factory = new EnrichProcessorFactory(null); + EnrichProcessorFactory factory = new EnrichProcessorFactory(null, scriptService); factory.metaData = createMetaData("majestic", policy); Map config = new HashMap<>(); @@ -81,7 +91,7 @@ public void testCreateProcessorInstance() throws Exception { public void testPolicyDoesNotExist() { List enrichValues = List.of("globalRank", "tldRank", "tld"); - EnrichProcessorFactory factory = new EnrichProcessorFactory(null); + EnrichProcessorFactory factory = new EnrichProcessorFactory(null, scriptService); factory.metaData = MetaData.builder().build(); Map config = new HashMap<>(); @@ -110,7 +120,7 @@ public void testPolicyDoesNotExist() { public void testPolicyNameMissing() { List enrichValues = List.of("globalRank", "tldRank", "tld"); - EnrichProcessorFactory factory = new EnrichProcessorFactory(null); + EnrichProcessorFactory factory = new EnrichProcessorFactory(null, scriptService); Map config = new HashMap<>(); config.put("enrich_key", "host"); @@ -138,7 +148,7 @@ public void testPolicyNameMissing() { public void testUnsupportedPolicy() throws Exception { List enrichValues = List.of("globalRank", "tldRank", "tld"); EnrichPolicy policy = new EnrichPolicy("unsupported", null, List.of("source_index"), "my_key", enrichValues); - EnrichProcessorFactory factory = new EnrichProcessorFactory(null); + EnrichProcessorFactory factory = new EnrichProcessorFactory(null, scriptService); factory.metaData = createMetaData("majestic", policy); Map config = new HashMap<>(); @@ -157,7 +167,7 @@ public void testUnsupportedPolicy() throws Exception { public void testCompactEnrichValuesFormat() throws Exception { List enrichValues = List.of("globalRank", "tldRank", "tld"); EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("source_index"), "host", enrichValues); - EnrichProcessorFactory factory = new EnrichProcessorFactory(null); + EnrichProcessorFactory factory = new EnrichProcessorFactory(null, scriptService); factory.metaData = createMetaData("majestic", policy); Map config = new HashMap<>(); @@ -175,7 +185,7 @@ public void testCompactEnrichValuesFormat() throws Exception { public void testNoTargetField() throws Exception { List enrichValues = List.of("globalRank", "tldRank", "tld"); EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("source_index"), "host", enrichValues); - EnrichProcessorFactory factory = new EnrichProcessorFactory(null); + EnrichProcessorFactory factory = new EnrichProcessorFactory(null, scriptService); factory.metaData = createMetaData("majestic", policy); Map config1 = new HashMap<>(); @@ -189,7 +199,7 @@ public void testNoTargetField() throws Exception { public void testIllegalMaxMatches() throws Exception { List enrichValues = List.of("globalRank", "tldRank", "tld"); EnrichPolicy policy = new EnrichPolicy(EnrichPolicy.MATCH_TYPE, null, List.of("source_index"), "my_key", enrichValues); - EnrichProcessorFactory factory = new EnrichProcessorFactory(null); + EnrichProcessorFactory factory = new EnrichProcessorFactory(null, scriptService); factory.metaData = createMetaData("majestic", policy); Map config = new HashMap<>(); diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java index 12833f5418d6c..b2459f1b5aa7a 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/GeoMatchProcessorTests.java @@ -37,6 +37,7 @@ import java.util.Map; import java.util.function.BiConsumer; +import static org.elasticsearch.xpack.enrich.MatchProcessorTests.str; import static org.hamcrest.Matchers.emptyArray; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -66,8 +67,8 @@ private void testBasicsForFieldValue(Object fieldValue, Geometry expectedGeometr "_tag", mockSearch, "_name", - "location", - "entry", + str("location"), + str("entry"), false, false, "shape", diff --git a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java index b96d3ed996109..ec8a5819d786a 100644 --- a/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java +++ b/x-pack/plugin/enrich/src/test/java/org/elasticsearch/xpack/enrich/MatchProcessorTests.java @@ -21,6 +21,8 @@ import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.query.TermsQueryBuilder; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.TestTemplateService; +import org.elasticsearch.script.TemplateScript; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregations; @@ -47,7 +49,17 @@ public class MatchProcessorTests extends ESTestCase { public void testBasics() throws Exception { int maxMatches = randomIntBetween(1, 8); MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank", 23, "tld", "co"))); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", true, false, "domain", maxMatches); + MatchProcessor processor = new MatchProcessor( + "_tag", + mockSearch, + "_name", + str("domain"), + str("entry"), + true, + false, + "domain", + maxMatches + ); IngestDocument ingestDocument = new IngestDocument( "_index", "_id", @@ -91,7 +103,7 @@ public void testBasics() throws Exception { public void testNoMatch() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", true, false, "domain", 1); + MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), true, false, "domain", 1); IngestDocument ingestDocument = new IngestDocument( "_index", "_id", @@ -127,7 +139,7 @@ public void testNoMatch() throws Exception { public void testSearchFailure() throws Exception { String indexName = ".enrich-_name"; MockSearchFunction mockSearch = mockedSearchFunction(new IndexNotFoundException(indexName)); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", true, false, "domain", 1); + MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), true, false, "domain", 1); IngestDocument ingestDocument = new IngestDocument( "_index", "_id", @@ -171,8 +183,8 @@ public void testIgnoreKeyMissing() throws Exception { "_tag", mockedSearchFunction(), "_name", - "domain", - "entry", + str("domain"), + str("entry"), true, true, "domain", @@ -191,8 +203,8 @@ public void testIgnoreKeyMissing() throws Exception { "_tag", mockedSearchFunction(), "_name", - "domain", - "entry", + str("domain"), + str("entry"), true, false, "domain", @@ -213,7 +225,7 @@ public void testIgnoreKeyMissing() throws Exception { public void testExistingFieldWithOverrideDisabled() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank", 23, "tld", "co"))); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", false, false, "domain", 1); + MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), false, false, "domain", 1); IngestDocument ingestDocument = new IngestDocument(new HashMap<>(Map.of("domain", "elastic.co", "tld", "tld")), Map.of()); IngestDocument[] resultHolder = new IngestDocument[1]; @@ -229,7 +241,7 @@ public void testExistingFieldWithOverrideDisabled() throws Exception { public void testExistingNullFieldWithOverrideDisabled() throws Exception { MockSearchFunction mockSearch = mockedSearchFunction(Map.of("elastic.co", Map.of("globalRank", 451, "tldRank", 23, "tld", "co"))); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", false, false, "domain", 1); + MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), false, false, "domain", 1); Map source = new HashMap<>(); source.put("domain", "elastic.co"); @@ -248,7 +260,7 @@ public void testExistingNullFieldWithOverrideDisabled() throws Exception { public void testNumericValue() { MockSearchFunction mockSearch = mockedSearchFunction(Map.of(2, Map.of("globalRank", 451, "tldRank", 23, "tld", "co"))); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", false, true, "domain", 1); + MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), false, true, "domain", 1); IngestDocument ingestDocument = new IngestDocument("_index", "_id", "_routing", 1L, VersionType.INTERNAL, Map.of("domain", 2)); // Execute @@ -276,7 +288,7 @@ public void testArray() { MockSearchFunction mockSearch = mockedSearchFunction( Map.of(List.of("1", "2"), Map.of("globalRank", 451, "tldRank", 23, "tld", "co")) ); - MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", "domain", "entry", false, true, "domain", 1); + MatchProcessor processor = new MatchProcessor("_tag", mockSearch, "_name", str("domain"), str("entry"), false, true, "domain", 1); IngestDocument ingestDocument = new IngestDocument( "_index", "_id", @@ -385,4 +397,8 @@ public SearchResponse mockResponse(Map> documents) { new SearchResponse.Clusters(1, 1, 0) ); } + + static TemplateScript.Factory str(String stringLiteral) { + return new TestTemplateService.MockTemplateScript.Factory(stringLiteral); + } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java index b06abe5cf677c..b6ac92134723c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessor.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.core.ml.inference.TrainedModelConfig; import org.elasticsearch.xpack.core.ml.inference.TrainedModelDefinition; import org.elasticsearch.xpack.core.ml.inference.TrainedModelInput; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import org.elasticsearch.xpack.ml.dataframe.DataFrameAnalyticsTask.ProgressTracker; import org.elasticsearch.xpack.ml.dataframe.process.results.AnalyticsResult; import org.elasticsearch.xpack.ml.dataframe.process.results.RowResults; @@ -37,6 +38,18 @@ public class AnalyticsResultProcessor { private static final Logger LOGGER = LogManager.getLogger(AnalyticsResultProcessor.class); + /** + * While we report progress as we read row results there are other things we need to account for + * to report completion. There are other types of results we can't predict the number of like + * progress objects and the inference model. Thus, we report a max progress until we know we have + * completed processing results. + * + * It is critical to ensure we do not report complete progress too soon as restarting a job + * uses the progress to determine which state to restart from. If we report full progress too soon + * we cannot restart a job as we will think the job was finished. + */ + private static final int MAX_PROGRESS_BEFORE_COMPLETION = 98; + private final DataFrameAnalyticsConfig analytics; private final DataFrameRowsJoiner dataFrameRowsJoiner; private final ProgressTracker progressTracker; @@ -68,7 +81,7 @@ public void awaitForCompletion() { completionLatch.await(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - LOGGER.error(new ParameterizedMessage("[{}] Interrupted waiting for results processor to complete", analytics.getId()), e); + setAndReportFailure(ExceptionsHelper.serverError("interrupted waiting for results processor to complete", e)); } } @@ -91,27 +104,32 @@ public void process(AnalyticsProcess process) { processResult(result, resultsJoiner); if (result.getRowResults() != null) { processedRows++; - progressTracker.writingResultsPercent.set(processedRows >= totalRows ? 100 : (int) (processedRows * 100.0 / totalRows)); + updateResultsProgress(processedRows >= totalRows ? 100 : (int) (processedRows * 100.0 / totalRows)); } } - if (isCancelled == false) { - // This means we completed successfully so we need to set the progress to 100. - // This is because due to skipped rows, it is possible the processed rows will not reach the total rows. - progressTracker.writingResultsPercent.set(100); - } } catch (Exception e) { if (isCancelled) { // No need to log error as it's due to stopping } else { - LOGGER.error(new ParameterizedMessage("[{}] Error parsing data frame analytics output", analytics.getId()), e); - failure = "error parsing data frame analytics output: [" + e.getMessage() + "]"; + setAndReportFailure(e); } } finally { + if (isCancelled == false && failure == null) { + completeResultsProgress(); + } completionLatch.countDown(); process.consumeAndCloseOutputStream(); } } + private void updateResultsProgress(int progress) { + progressTracker.writingResultsPercent.set(Math.min(progress, MAX_PROGRESS_BEFORE_COMPLETION)); + } + + private void completeResultsProgress() { + progressTracker.writingResultsPercent.set(100); + } + private void processResult(AnalyticsResult result, DataFrameRowsJoiner resultsJoiner) { RowResults rowResults = result.getRowResults(); if (rowResults != null) { @@ -137,7 +155,7 @@ private void createAndIndexInferenceModel(TrainedModelDefinition.Builder inferen } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - LOGGER.error(new ParameterizedMessage("[{}] Interrupted waiting for inference model to be stored", analytics.getId()), e); + setAndReportFailure(ExceptionsHelper.serverError("interrupted waiting for inference model to be stored")); } } @@ -168,19 +186,22 @@ private CountDownLatch storeTrainedModel(TrainedModelConfig trainedModelConfig) aBoolean -> { if (aBoolean == false) { LOGGER.error("[{}] Storing trained model responded false", analytics.getId()); + setAndReportFailure(ExceptionsHelper.serverError("storing trained model responded false")); } else { LOGGER.info("[{}] Stored trained model with id [{}]", analytics.getId(), trainedModelConfig.getModelId()); auditor.info(analytics.getId(), "Stored trained model with id [" + trainedModelConfig.getModelId() + "]"); } }, - e -> { - LOGGER.error(new ParameterizedMessage("[{}] Error storing trained model [{}]", analytics.getId(), - trainedModelConfig.getModelId()), e); - auditor.error(analytics.getId(), "Error storing trained model with id [" + trainedModelConfig.getModelId() - + "]; error message [" + e.getMessage() + "]"); - } + e -> setAndReportFailure(ExceptionsHelper.serverError("error storing trained model with id [{}]", e, + trainedModelConfig.getModelId())) ); trainedModelProvider.storeTrainedModel(trainedModelConfig, new LatchedActionListener<>(storeListener, latch)); return latch; } + + private void setAndReportFailure(Exception e) { + LOGGER.error(new ParameterizedMessage("[{}] Error processing results; ", analytics.getId()), e); + failure = "error processing results; " + e.getMessage(); + auditor.error(analytics.getId(), "Error processing results; " + e.getMessage()); + } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java index aaa64c13a1c44..0d2b5aea364eb 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/dataframe/process/AnalyticsResultProcessorTests.java @@ -39,9 +39,11 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.startsWith; import static org.mockito.Matchers.any; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -117,6 +119,28 @@ public void testProcess_GivenRowResults() { assertThat(progressTracker.writingResultsPercent.get(), equalTo(100)); } + public void testProcess_GivenDataFrameRowsJoinerFails() { + givenDataFrameRows(2); + RowResults rowResults1 = mock(RowResults.class); + RowResults rowResults2 = mock(RowResults.class); + givenProcessResults(Arrays.asList(new AnalyticsResult(rowResults1, 50, null), new AnalyticsResult(rowResults2, 100, null))); + + doThrow(new RuntimeException("some failure")).when(dataFrameRowsJoiner).processRowResults(any(RowResults.class)); + + AnalyticsResultProcessor resultProcessor = createResultProcessor(); + + resultProcessor.process(process); + resultProcessor.awaitForCompletion(); + + assertThat(resultProcessor.getFailure(), equalTo("error processing results; some failure")); + + ArgumentCaptor auditCaptor = ArgumentCaptor.forClass(String.class); + verify(auditor).error(eq(JOB_ID), auditCaptor.capture()); + assertThat(auditCaptor.getValue(), containsString("Error processing results; some failure")); + + assertThat(progressTracker.writingResultsPercent.get(), equalTo(0)); + } + @SuppressWarnings("unchecked") public void testProcess_GivenInferenceModelIsStoredSuccessfully() { givenDataFrameRows(0); @@ -182,9 +206,11 @@ public void testProcess_GivenInferenceModelFailedToStore() { // This test verifies the processor knows how to handle a failure on storing the model and completes normally ArgumentCaptor auditCaptor = ArgumentCaptor.forClass(String.class); verify(auditor).error(eq(JOB_ID), auditCaptor.capture()); - assertThat(auditCaptor.getValue(), containsString("Error storing trained model with id [" + JOB_ID)); - assertThat(auditCaptor.getValue(), containsString("[some failure]")); + assertThat(auditCaptor.getValue(), containsString("Error processing results; error storing trained model with id [" + JOB_ID)); Mockito.verifyNoMoreInteractions(auditor); + + assertThat(resultProcessor.getFailure(), startsWith("error processing results; error storing trained model with id [" + JOB_ID)); + assertThat(progressTracker.writingResultsPercent.get(), equalTo(0)); } private void givenProcessResults(List results) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java index f5175b526be12..20289c5f09e91 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/AuthenticationService.java @@ -116,7 +116,7 @@ public AuthenticationService(Settings settings, Realms realms, AuditTrailService * a user was indeed associated with the request and the credentials were verified to be valid), the method returns * the user and that user is then "attached" to the request's context. * - * @param request The request to be authenticated + * @param request The request to be authenticated */ public void authenticate(RestRequest request, ActionListener authenticationListener) { createAuthenticator(request, authenticationListener).authenticateAsync(); @@ -128,12 +128,12 @@ public void authenticate(RestRequest request, ActionListener aut * the user and that user is then "attached" to the message's context. If no user was found to be attached to the given * message, then the given fallback user will be returned instead. * - * @param action The action of the message - * @param message The message to be authenticated - * @param fallbackUser The default user that will be assumed if no other user is attached to the message. Can be - * {@code null}, in which case there will be no fallback user and the success/failure of the - * authentication will be based on the whether there's an attached user to in the message and - * if there is, whether its credentials are valid. + * @param action The action of the message + * @param message The message to be authenticated + * @param fallbackUser The default user that will be assumed if no other user is attached to the message. Can be + * {@code null}, in which case there will be no fallback user and the success/failure of the + * authentication will be based on the whether there's an attached user to in the message and + * if there is, whether its credentials are valid. */ public void authenticate(String action, TransportMessage message, User fallbackUser, ActionListener listener) { createAuthenticator(action, message, fallbackUser, listener).authenticateAsync(); @@ -226,23 +226,25 @@ private Authenticator(AuditableRequest auditableRequest, User fallbackUser, Acti * these operations are: * *
    - *
  1. look for existing authentication {@link #lookForExistingAuthentication(Consumer)}
    2. - *
  2. look for a user token
    3. - *
  3. token extraction {@link #extractToken(Consumer)}
    4. - *
  4. token authentication {@link #consumeToken(AuthenticationToken)}
    5. - *
  5. user lookup for run as if necessary {@link #consumeUser(User, Map)} and - * {@link #lookupRunAsUser(User, String, Consumer)}
    6. - *
  6. write authentication into the context {@link #finishAuthentication(User)}
    7. + *
  7. look for existing authentication {@link #lookForExistingAuthentication(Consumer)}
    8. + *
  8. look for a user token
    9. + *
  9. token extraction {@link #extractToken(Consumer)}
    10. + *
  10. token authentication {@link #consumeToken(AuthenticationToken)}
    11. + *
  11. user lookup for run as if necessary {@link #consumeUser(User, Map)} and + * {@link #lookupRunAsUser(User, String, Consumer)}
    12. + *
  12. write authentication into the context {@link #finishAuthentication(User)}
    13. *
*/ private void authenticateAsync() { if (defaultOrderedRealmList.isEmpty()) { // this happens when the license state changes between the call to authenticate and the actual invocation // to get the realm list + logger.debug("No realms available, failing authentication"); listener.onResponse(null); } else { lookForExistingAuthentication((authentication) -> { if (authentication != null) { + logger.trace("Found existing authentication [{}] in request [{}]", authentication, request); listener.onResponse(authentication); } else { tokenService.getAndValidateToken(threadContext, ActionListener.wrap(userToken -> { @@ -252,6 +254,7 @@ private void authenticateAsync() { checkForApiKey(); } }, e -> { + logger.debug(new ParameterizedMessage("Failed to validate token authentication for request [{}]", request), e); if (e instanceof ElasticsearchSecurityException && tokenService.isExpiredTokenException((ElasticsearchSecurityException) e) == false) { // intentionally ignore the returned exception; we call this primarily @@ -275,6 +278,7 @@ private void checkForApiKey() { } else if (authResult.getStatus() == AuthenticationResult.Status.TERMINATE) { Exception e = (authResult.getException() != null) ? authResult.getException() : Exceptions.authenticationError(authResult.getMessage()); + logger.debug(new ParameterizedMessage("API key service terminated authentication for request [{}]", request), e); listener.onFailure(e); } else { if (authResult.getMessage() != null) { @@ -308,7 +312,7 @@ private void lookForExistingAuthentication(Consumer authenticati } catch (Exception e) { logger.error((Supplier) () -> new ParameterizedMessage("caught exception while trying to read authentication from request [{}]", request), - e); + e); action = () -> listener.onFailure(request.tamperedRequest()); } @@ -332,6 +336,8 @@ void extractToken(Consumer consumer) { for (Realm realm : defaultOrderedRealmList) { final AuthenticationToken token = realm.token(threadContext); if (token != null) { + logger.trace("Found authentication credentials [{}] for principal [{}] in request [{}]", + token.getClass().getName(), token.principal(), request); action = () -> consumer.accept(token); break; } @@ -358,12 +364,17 @@ private void consumeToken(AuthenticationToken token) { } else { authenticationToken = token; final List realmsList = getRealmList(authenticationToken.principal()); + logger.trace("Checking token of type [{}] against [{}] realm(s)", token.getClass().getName(), realmsList.size()); final long startInvalidation = numInvalidation.get(); final Map> messages = new LinkedHashMap<>(); final BiConsumer> realmAuthenticatingConsumer = (realm, userListener) -> { if (realm.supports(authenticationToken)) { + logger.trace("Trying to authenticate [{}] using realm [{}] with token [{}] ", + token.principal(), realm, token.getClass().getName()); realm.authenticate(authenticationToken, ActionListener.wrap((result) -> { assert result != null : "Realm " + realm + " produced a null authentication result"; + logger.debug("Authentication of [{}] using realm [{}] with token [{}] was [{}]", + token.principal(), realm, token.getClass().getSimpleName(), result); if (result.getStatus() == AuthenticationResult.Status.SUCCESS) { // user was authenticated, populate the authenticated by information authenticatedBy = new RealmRef(realm.name(), realm.type(), nodeName); @@ -377,9 +388,9 @@ private void consumeToken(AuthenticationToken token) { request.realmAuthenticationFailed(authenticationToken, realm.name()); if (result.getStatus() == AuthenticationResult.Status.TERMINATE) { logger.info("Authentication of [{}] was terminated by realm [{}] - {}", - authenticationToken.principal(), realm.name(), result.getMessage()); + authenticationToken.principal(), realm.name(), result.getMessage()); Exception e = (result.getException() != null) ? result.getException() - : Exceptions.authenticationError(result.getMessage()); + : Exceptions.authenticationError(result.getMessage()); userListener.onFailure(e); } else { if (result.getMessage() != null) { @@ -390,8 +401,8 @@ private void consumeToken(AuthenticationToken token) { } }, (ex) -> { logger.warn(new ParameterizedMessage( - "An error occurred while attempting to authenticate [{}] against realm [{}]", - authenticationToken.principal(), realm.name()), ex); + "An error occurred while attempting to authenticate [{}] against realm [{}]", + authenticationToken.principal(), realm.name()), ex); userListener.onFailure(ex); })); } else { @@ -407,6 +418,8 @@ private void consumeToken(AuthenticationToken token) { try { authenticatingListener.run(); } catch (Exception e) { + logger.debug(new ParameterizedMessage("Authentication of [{}] with token [{}] failed", + token.principal(), token.getClass().getName()), e); listener.onFailure(request.exceptionProcessingRequest(e, token)); } } @@ -427,11 +440,13 @@ private List getRealmList(String principal) { if (index > 0) { final List smartOrder = new ArrayList<>(orderedRealmList.size()); smartOrder.add(lastSuccess); - for (int i = 1; i < orderedRealmList.size(); i++) { + for (int i = 0; i < orderedRealmList.size(); i++) { if (i != index) { smartOrder.add(orderedRealmList.get(i)); } } + assert smartOrder.size() == orderedRealmList.size() && smartOrder.containsAll(orderedRealmList) + : "Element mismatch between SmartOrder=" + smartOrder + " and DefaultOrder=" + orderedRealmList; return Collections.unmodifiableList(smartOrder); } } @@ -443,23 +458,25 @@ private List getRealmList(String principal) { * Handles failed extraction of an authentication token. This can happen in a few different scenarios: * *
    - *
  • this is an initial request from a client without preemptive authentication, so we must return an authentication - * challenge
    • - *
  • this is a request made internally within a node and there is a fallback user, which is typically the - * {@link SystemUser}
    • - *
  • anonymous access is enabled and this will be considered an anonymous request
    • + *
  • this is an initial request from a client without preemptive authentication, so we must return an authentication + * challenge
    • + *
  • this is a request made internally within a node and there is a fallback user, which is typically the + * {@link SystemUser}
    • + *
  • anonymous access is enabled and this will be considered an anonymous request
    • *
- * + *

* Regardless of the scenario, this method will call the listener with either failure or success. */ // pkg-private for tests void handleNullToken() { final Authentication authentication; if (fallbackUser != null) { + logger.trace("No valid credentials found in request [{}], using fallback [{}]", request, fallbackUser.principal()); RealmRef authenticatedBy = new RealmRef("__fallback", "__fallback", nodeName); authentication = new Authentication(fallbackUser, authenticatedBy, null, Version.CURRENT, AuthenticationType.INTERNAL, Collections.emptyMap()); } else if (isAnonymousUserEnabled) { + logger.trace("No valid credentials found in request [{}], using anonymous [{}]", request, anonymousUser.principal()); RealmRef authenticatedBy = new RealmRef("__anonymous", "__anonymous", nodeName); authentication = new Authentication(anonymousUser, authenticatedBy, null, Version.CURRENT, AuthenticationType.ANONYMOUS, Collections.emptyMap()); @@ -471,7 +488,10 @@ void handleNullToken() { if (authentication != null) { action = () -> writeAuthToContext(authentication); } else { - action = () -> listener.onFailure(request.anonymousAccessDenied()); + action = () -> { + logger.debug("No valid credentials found in request [{}], rejecting", request); + listener.onFailure(request.anonymousAccessDenied()); + }; } // we assign the listener call to an action to avoid calling the listener within a try block and auditing the wrong thing when @@ -499,6 +519,7 @@ private void consumeUser(User user, Map> message Strings.collectionToCommaDelimitedString(defaultOrderedRealmList), Strings.collectionToCommaDelimitedString(unlicensedRealms)); } + logger.trace("Failed to authenticate request [{}]", request); listener.onFailure(request.authenticationFailed(authenticationToken)); } else { threadContext.putTransient(AuthenticationResult.THREAD_CONTEXT_KEY, authenticationResult); @@ -512,7 +533,7 @@ private void consumeUser(User user, Map> message assert runAsUsername.isEmpty() : "the run as username may not be empty"; logger.debug("user [{}] attempted to runAs with an empty username", user.principal()); listener.onFailure(request.runAsDenied( - new Authentication(new User(runAsUsername, null, user), authenticatedBy, lookedupBy), authenticationToken)); + new Authentication(new User(runAsUsername, null, user), authenticatedBy, lookedupBy), authenticationToken)); } } else { finishAuthentication(user); @@ -526,10 +547,12 @@ private void consumeUser(User user, Map> message * names of users that exist using a timing attack */ private void lookupRunAsUser(final User user, String runAsUsername, Consumer userConsumer) { + logger.trace("Looking up run-as user [{}] for authenticated user [{}]", runAsUsername, user.principal()); final RealmUserLookup lookup = new RealmUserLookup(getRealmList(runAsUsername), threadContext); final long startInvalidationNum = numInvalidation.get(); lookup.lookup(runAsUsername, ActionListener.wrap(tuple -> { if (tuple == null) { + logger.debug("Cannot find run-as user [{}] for authenticated user [{}]", runAsUsername, user.principal()); // the user does not exist, but we still create a User object, which will later be rejected by authz userConsumer.accept(new User(runAsUsername, null, user)); } else { @@ -541,6 +564,7 @@ private void lookupRunAsUser(final User user, String runAsUsername, Consumer realm); } + logger.trace("Using run-as user [{}] with authenticated user [{}]", foundUser, user.principal()); userConsumer.accept(new User(foundUser, user)); } }, exception -> listener.onFailure(request.exceptionProcessingRequest(exception, authenticationToken)))); @@ -567,11 +591,18 @@ void finishAuthentication(User finalUser) { */ void writeAuthToContext(Authentication authentication) { request.authenticationSuccess(authentication.getAuthenticatedBy().getName(), authentication.getUser()); - Runnable action = () -> listener.onResponse(authentication); + Runnable action = () -> { + logger.trace("Established authentication [{}] for request [{}]", authentication, request); + listener.onResponse(authentication); + }; try { authentication.writeToContext(threadContext); } catch (Exception e) { - action = () -> listener.onFailure(request.exceptionProcessingRequest(e, authenticationToken)); + action = () -> { + logger.debug( + new ParameterizedMessage("Failed to store authentication [{}] for request [{}]", authentication, request), e); + listener.onFailure(request.exceptionProcessingRequest(e, authenticationToken)); + }; } // we assign the listener call to an action to avoid calling the listener within a try block and auditing the wrong thing diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java index 6a5f49545933f..df678f9c63ba4 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/rest/SecurityRestFilter.java @@ -54,9 +54,15 @@ public void handleRequest(RestRequest request, RestChannel channel, NodeClient c } service.authenticate(maybeWrapRestRequest(request), ActionListener.wrap( authentication -> { + if (authentication == null) { + logger.trace("No authentication available for REST request [{}]", request.uri()); + } else { + logger.trace("Authenticated REST request [{}] as {}", request.uri(), authentication); + } RemoteHostHeader.process(request, threadContext); restHandler.handleRequest(request, channel, client); }, e -> { + logger.debug(new ParameterizedMessage("Authentication failed for REST request [{}]", request.uri()), e); try { channel.sendResponse(new BytesRestResponse(channel, e)); } catch (Exception inner) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java index 8214f74b40227..da60e053f0815 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentAndFieldLevelSecurityTests.java @@ -76,7 +76,7 @@ protected String configRoles() { " - names: '*'\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field1 ]\n" + + " grant: [ field1, id ]\n" + " query: '{\"term\" : {\"field1\" : \"value1\"}}'\n" + "role3:\n" + " cluster: [ all ]\n" + @@ -84,7 +84,7 @@ protected String configRoles() { " - names: '*'\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field2 ]\n" + + " grant: [ field2, id ]\n" + " query: '{\"term\" : {\"field2\" : \"value2\"}}'\n" + "role4:\n" + " cluster: [ all ]\n" + @@ -92,7 +92,7 @@ protected String configRoles() { " - names: '*'\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field1 ]\n" + + " grant: [ field1, id ]\n" + " query: '{\"term\" : {\"field2\" : \"value2\"}}'\n"; } @@ -106,12 +106,12 @@ public Settings nodeSettings(int nodeOrdinal) { public void testSimpleQuery() { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type1", "field1", "type=text", "field2", "type=text") + .addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text") ); - client().prepareIndex("test").setId("1").setSource("field1", "value1") + client().prepareIndex("test").setId("1").setSource("id", "1", "field1", "value1") .setRefreshPolicy(IMMEDIATE) .get(); - client().prepareIndex("test").setId("2").setSource("field2", "value2") + client().prepareIndex("test").setId("2").setSource("id", "2", "field2", "value2") .setRefreshPolicy(IMMEDIATE) .get(); @@ -121,20 +121,22 @@ public void testSimpleQuery() { .get(); assertHitCount(response, 1); assertSearchHits(response, "1"); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1").toString(), equalTo("value1")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id").toString(), equalTo("1")); response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))) .prepareSearch("test") .get(); assertHitCount(response, 1); assertSearchHits(response, "2"); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field2").toString(), equalTo("value2")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id").toString(), equalTo("2")); response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertSearchHits(response, "1", "2"); @@ -172,12 +174,12 @@ public void testDLSIsAppliedBeforeFLS() { public void testQueryCache() { assertAcked(client().admin().indices().prepareCreate("test") .setSettings(Settings.builder().put(IndexModule.INDEX_QUERY_CACHE_EVERYTHING_SETTING.getKey(), true)) - .addMapping("type1", "field1", "type=text", "field2", "type=text") + .addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text") ); - client().prepareIndex("test").setId("1").setSource("field1", "value1") + client().prepareIndex("test").setId("1").setSource("id", "1", "field1", "value1") .setRefreshPolicy(IMMEDIATE) .get(); - client().prepareIndex("test").setId("2").setSource("field2", "value2") + client().prepareIndex("test").setId("2").setSource("id", "2", "field2", "value2") .setRefreshPolicy(IMMEDIATE) .get(); @@ -190,15 +192,17 @@ public void testQueryCache() { .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), equalTo("value1")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("1")); response = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))) .prepareSearch("test") .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("2")); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field2"), equalTo("value2")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("2")); // this is a bit weird the document level permission (all docs with field2:value2) don't match with the field level // permissions (field1), @@ -208,21 +212,24 @@ public void testQueryCache() { .get(); assertHitCount(response, 1); assertThat(response.getHits().getAt(0).getId(), equalTo("2")); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(0)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("2")); // user4 has all roles response = client().filterWithHeader( Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(response, 2); assertThat(response.getHits().getAt(0).getId(), equalTo("1")); - assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(0).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(0).getSourceAsMap().get("field1"), equalTo("value1")); + assertThat(response.getHits().getAt(0).getSourceAsMap().get("id"), equalTo("1")); assertThat(response.getHits().getAt(1).getId(), equalTo("2")); - assertThat(response.getHits().getAt(1).getSourceAsMap().size(), equalTo(1)); + assertThat(response.getHits().getAt(1).getSourceAsMap().size(), equalTo(2)); assertThat(response.getHits().getAt(1).getSourceAsMap().get("field2"), equalTo("value2")); + assertThat(response.getHits().getAt(1).getSourceAsMap().get("id"), equalTo("2")); } } @@ -319,7 +326,7 @@ public void testGetFieldMappingsIsFiltered() { Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user1", USERS_PASSWD))) .admin().indices().prepareGetFieldMappings("test").setFields("*").get(); - Map>> mappings = + Map> mappings = getFieldMappingsResponse.mappings(); assertEquals(1, mappings.size()); assertExpectedFields(mappings.get("test"), "field1"); @@ -329,7 +336,7 @@ public void testGetFieldMappingsIsFiltered() { Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))) .admin().indices().prepareGetFieldMappings("test").setFields("*").get(); - Map>> mappings = + Map> mappings = getFieldMappingsResponse.mappings(); assertEquals(1, mappings.size()); assertExpectedFields(mappings.get("test"), "field2"); @@ -339,7 +346,7 @@ public void testGetFieldMappingsIsFiltered() { Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD))) .admin().indices().prepareGetFieldMappings("test").setFields("*").get(); - Map>> mappings = + Map> mappings = getFieldMappingsResponse.mappings(); assertEquals(1, mappings.size()); assertExpectedFields(mappings.get("test"), "field1"); @@ -349,7 +356,7 @@ public void testGetFieldMappingsIsFiltered() { Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD))) .admin().indices().prepareGetFieldMappings("test").setFields("*").get(); - Map>> mappings = + Map> mappings = getFieldMappingsResponse.mappings(); assertEquals(1, mappings.size()); assertExpectedFields(mappings.get("test"), "field1", "field2"); @@ -423,11 +430,10 @@ private static void assertExpectedFields(FieldCapabilitiesResponse fieldCapabili assertEquals("Some unexpected fields were returned: " + responseMap.keySet(), 0, responseMap.size()); } - private static void assertExpectedFields(Map> mappings, + private static void assertExpectedFields(Map actual, String... expectedFields) { - assertEquals(1, mappings.size()); - Map fields = new HashMap<>(mappings.get("_doc")); Set builtInMetaDataFields = IndicesModule.getBuiltInMetaDataFields(); + Map fields = new HashMap<>(actual); for (String field : builtInMetaDataFields) { GetFieldMappingsResponse.FieldMappingMetaData fieldMappingMetaData = fields.remove(field); assertNotNull(" expected field [" + field + "] not found", fieldMappingMetaData); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java index f23d84b9ad8a1..f4b2556fb4816 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/DocumentLevelSecurityTests.java @@ -602,6 +602,9 @@ public void testGlobalAggregation() throws Exception { public void testParentChild() throws Exception { XContentBuilder mapping = jsonBuilder().startObject() .startObject("properties") + .startObject("id") + .field("type", "keyword") + .endObject() .startObject("join_field") .field("type", "join") .startObject("relations") @@ -628,15 +631,18 @@ public void testParentChild() throws Exception { Map source = new HashMap<>(); source.put("field2", "value2"); + source.put("id", "c1"); Map joinField = new HashMap<>(); joinField.put("name", "child"); joinField.put("parent", "p1"); source.put("join_field", joinField); client().prepareIndex("test").setId("c1").setSource(source).setRouting("p1").get(); + source.put("id", "c2"); client().prepareIndex("test").setId("c2").setSource(source).setRouting("p1").get(); source = new HashMap<>(); source.put("field3", "value3"); source.put("join_field", joinField); + source.put("id", "c3"); client().prepareIndex("test").setId("c3").setSource(source).setRouting("p1").get(); refresh(); verifyParentChild(); @@ -651,7 +657,7 @@ private void verifyParentChild() { searchResponse = client().prepareSearch("test") .setQuery(hasParentQuery("parent", matchAllQuery(), false)) - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .get(); assertHitCount(searchResponse, 3L); assertThat(searchResponse.getHits().getAt(0).getId(), equalTo("c1")); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java index 82ebe7c7d7489..a802c97a4647f 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/FieldLevelSecurityRandomTests.java @@ -64,6 +64,7 @@ protected String configRoles() { allowedFields = new HashSet<>(); disAllowedFields = new HashSet<>(); int numFields = scaledRandomIntBetween(5, 50); + allowedFields.add("id"); for (int i = 0; i < numFields; i++) { String field = "field" + i; if (i % 2 == 0) { @@ -100,21 +101,21 @@ protected String configRoles() { " privileges:\n" + " - all\n" + " field_security:\n" + - " grant: [ field1 ]\n" + + " grant: [ id, field1 ]\n" + "role4:\n" + " cluster: [ all ]\n" + " indices:\n" + " - names: test\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field2 ]\n" + + " grant: [ id, field2 ]\n" + "role5:\n" + " cluster: [ all ]\n" + " indices:\n" + " - names: test\n" + " privileges: [ ALL ]\n" + " field_security:\n" + - " grant: [ field3 ]\n"; + " grant: [ id, field3 ]\n"; } @Override @@ -166,7 +167,7 @@ public void testRandom() throws Exception { public void testDuel() throws Exception { assertAcked(client().admin().indices().prepareCreate("test") - .addMapping("type1", "field1", "type=text", "field2", "type=text", "field3", "type=text") + .addMapping("type1", "id", "type=keyword", "field1", "type=text", "field2", "type=text", "field3", "type=text") ); int numDocs = scaledRandomIntBetween(32, 128); @@ -174,14 +175,14 @@ public void testDuel() throws Exception { for (int i = 1; i <= numDocs; i++) { String field = randomFrom("field1", "field2", "field3"); String value = "value"; - requests.add(client().prepareIndex("test").setId(value).setSource(field, value)); + requests.add(client().prepareIndex("test").setId(value).setSource("id", Integer.toString(i), field, value)); } indexRandom(true, requests); SearchResponse actual = client() .filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user2", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field1", "value")) .should(QueryBuilders.termQuery("field2", "value")) @@ -189,7 +190,7 @@ public void testDuel() throws Exception { ) .get(); SearchResponse expected = client().prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field1", "value")) ) @@ -202,7 +203,7 @@ public void testDuel() throws Exception { actual = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user3", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field1", "value")) .should(QueryBuilders.termQuery("field2", "value")) @@ -210,7 +211,7 @@ public void testDuel() throws Exception { ) .get(); expected = client().prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field2", "value")) ) @@ -223,7 +224,7 @@ public void testDuel() throws Exception { actual = client().filterWithHeader(Collections.singletonMap(BASIC_AUTH_HEADER, basicAuthHeaderValue("user4", USERS_PASSWD))) .prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field1", "value")) .should(QueryBuilders.termQuery("field2", "value")) @@ -231,7 +232,7 @@ public void testDuel() throws Exception { ) .get(); expected = client().prepareSearch("test") - .addSort("_id", SortOrder.ASC) + .addSort("id", SortOrder.ASC) .setQuery(QueryBuilders.boolQuery() .should(QueryBuilders.termQuery("field3", "value")) ) diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java index e99abb225defa..4e10bde26095b 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/integration/KibanaUserRoleIntegTests.java @@ -61,24 +61,21 @@ public String configUsersRoles() { public void testFieldMappings() throws Exception { final String index = "logstash-20-12-2015"; - final String type = "_doc"; final String field = "foo"; indexRandom(true, client().prepareIndex().setIndex(index).setSource(field, "bar")); GetFieldMappingsResponse response = client().admin().indices().prepareGetFieldMappings().addIndices("logstash-*").setFields("*") .includeDefaults(true).get(); - FieldMappingMetaData fieldMappingMetaData = response.fieldMappings(index, type, field); + FieldMappingMetaData fieldMappingMetaData = response.fieldMappings(index, field); assertThat(fieldMappingMetaData, notNullValue()); - assertThat(fieldMappingMetaData.isNull(), is(false)); response = client() .filterWithHeader(singletonMap("Authorization", UsernamePasswordToken.basicAuthHeaderValue("kibana_user", USERS_PASSWD))) .admin().indices().prepareGetFieldMappings().addIndices("logstash-*") .setFields("*") .includeDefaults(true).get(); - FieldMappingMetaData fieldMappingMetaData1 = response.fieldMappings(index, type, field); + FieldMappingMetaData fieldMappingMetaData1 = response.fieldMappings(index, field); assertThat(fieldMappingMetaData1, notNullValue()); - assertThat(fieldMappingMetaData1.isNull(), is(false)); assertThat(fieldMappingMetaData1.fullName(), equalTo(fieldMappingMetaData.fullName())); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java index 56b4ba8bb32ee..532b9121c1a00 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/AuthenticationServiceTests.java @@ -134,6 +134,10 @@ */ public class AuthenticationServiceTests extends ESTestCase { + private static final String SECOND_REALM_NAME = "second_realm"; + private static final String SECOND_REALM_TYPE = "second"; + private static final String FIRST_REALM_NAME = "file_realm"; + private static final String FIRST_REALM_TYPE = "file"; private AuthenticationService service; private TransportMessage message; private RestRequest restRequest; @@ -167,11 +171,11 @@ public void init() throws Exception { threadContext = new ThreadContext(Settings.EMPTY); firstRealm = mock(Realm.class); - when(firstRealm.type()).thenReturn("file"); - when(firstRealm.name()).thenReturn("file_realm"); + when(firstRealm.type()).thenReturn(FIRST_REALM_TYPE); + when(firstRealm.name()).thenReturn(FIRST_REALM_NAME); secondRealm = mock(Realm.class); - when(secondRealm.type()).thenReturn("second"); - when(secondRealm.name()).thenReturn("second_realm"); + when(secondRealm.type()).thenReturn(SECOND_REALM_TYPE); + when(secondRealm.name()).thenReturn(SECOND_REALM_NAME); Settings settings = Settings.builder() .put("path.home", createTempDir()) .put("node.name", "authc_test") @@ -304,26 +308,35 @@ public void testAuthenticateSmartRealmOrdering() { when(secondRealm.token(threadContext)).thenReturn(token); final String reqId = AuditUtil.getOrGenerateRequestId(threadContext); + // Authenticate against the normal chain. 1st Realm will be checked (and not pass) then 2nd realm will successfully authc final AtomicBoolean completed = new AtomicBoolean(false); service.authenticate("_action", message, (User)null, ActionListener.wrap(result -> { assertThat(result, notNullValue()); assertThat(result.getUser(), is(user)); assertThat(result.getLookedUpBy(), is(nullValue())); assertThat(result.getAuthenticatedBy(), is(notNullValue())); // TODO implement equals + assertThat(result.getAuthenticatedBy().getName(), is(SECOND_REALM_NAME)); + assertThat(result.getAuthenticatedBy().getType(), is(SECOND_REALM_TYPE)); assertThreadContextContainsAuthentication(result); setCompletedToTrue(completed); }, this::logAndFail)); assertTrue(completed.get()); completed.set(false); + // Authenticate against the smart chain. + // "SecondRealm" will be at the top of the list and will successfully authc. + // "FirstRealm" will not be used service.authenticate("_action", message, (User)null, ActionListener.wrap(result -> { assertThat(result, notNullValue()); assertThat(result.getUser(), is(user)); assertThat(result.getLookedUpBy(), is(nullValue())); assertThat(result.getAuthenticatedBy(), is(notNullValue())); // TODO implement equals + assertThat(result.getAuthenticatedBy().getName(), is(SECOND_REALM_NAME)); + assertThat(result.getAuthenticatedBy().getType(), is(SECOND_REALM_TYPE)); assertThreadContextContainsAuthentication(result); setCompletedToTrue(completed); }, this::logAndFail)); + verify(auditTrail).authenticationFailed(reqId, firstRealm.name(), token, "_action", message); verify(auditTrail, times(2)).authenticationSuccess(reqId, secondRealm.name(), user, "_action", message); verify(firstRealm, times(2)).name(); // used above one time @@ -336,6 +349,30 @@ public void testAuthenticateSmartRealmOrdering() { verify(firstRealm).authenticate(eq(token), any(ActionListener.class)); verify(secondRealm, times(2)).authenticate(eq(token), any(ActionListener.class)); verifyNoMoreInteractions(auditTrail, firstRealm, secondRealm); + + // Now assume some change in the backend system so that 2nd realm no longer has the user, but the 1st realm does. + mockAuthenticate(secondRealm, token, null); + mockAuthenticate(firstRealm, token, user); + + completed.set(false); + // This will authenticate against the smart chain. + // "SecondRealm" will be at the top of the list but will no longer authenticate the user. + // Then "FirstRealm" will be checked. + service.authenticate("_action", message, (User)null, ActionListener.wrap(result -> { + assertThat(result, notNullValue()); + assertThat(result.getUser(), is(user)); + assertThat(result.getLookedUpBy(), is(nullValue())); + assertThat(result.getAuthenticatedBy(), is(notNullValue())); + assertThat(result.getAuthenticatedBy().getName(), is(FIRST_REALM_NAME)); + assertThat(result.getAuthenticatedBy().getType(), is(FIRST_REALM_TYPE)); + assertThreadContextContainsAuthentication(result); + setCompletedToTrue(completed); + }, this::logAndFail)); + + verify(auditTrail, times(1)).authenticationFailed(reqId, SECOND_REALM_NAME, token, "_action", message); + verify(auditTrail, times(1)).authenticationSuccess(reqId, FIRST_REALM_NAME, user, "_action", message); + verify(secondRealm, times(3)).authenticate(eq(token), any(ActionListener.class)); // 2 from above + 1 more + verify(firstRealm, times(2)).authenticate(eq(token), any(ActionListener.class)); // 1 from above + 1 more } public void testCacheClearOnSecurityIndexChange() { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index b2915f48fce9b..e8585d9c6cb97 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -76,7 +76,7 @@ import java.util.Map; import static java.time.Clock.systemUTC; -import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; +import static org.elasticsearch.repositories.ESBlobStoreContainerTestCase.randomBytes; import static org.elasticsearch.test.ClusterServiceUtils.setState; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; diff --git a/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec index 19ee3d260b83c..182b6c2c76f39 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/agg.csv-spec @@ -534,6 +534,29 @@ SELECT HISTOGRAM(YEAR(birth_date), 2) AS h, COUNT(*) as c FROM test_emp GROUP BY null |10 ; +histogramYearOnDateTimeWithScalars +schema::year:i|c:l +SELECT YEAR(CAST(birth_date + INTERVAL 5 YEARS AS DATE) + INTERVAL 20 MONTHS) AS year, COUNT(*) as c FROM test_emp GROUP BY 1; + + year | c +---------------+--------------- +null |10 +1958 |2 +1959 |12 +1960 |7 +1961 |7 +1962 |4 +1963 |5 +1964 |5 +1965 |7 +1966 |9 +1967 |7 +1968 |7 +1969 |8 +1970 |6 +1971 |4 +; + histogramNumericWithExpression schema::h:i|c:l SELECT HISTOGRAM(emp_no % 100, 10) AS h, COUNT(*) as c FROM test_emp GROUP BY h ORDER BY h DESC; diff --git a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec index 1a4ce79d2c2fe..b17ba988472be 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/command.csv-spec @@ -122,8 +122,9 @@ SIN |SCALAR SINH |SCALAR SQRT |SCALAR TAN |SCALAR -TRUNCATE |SCALAR -ASCII |SCALAR +TRUNC |SCALAR +TRUNCATE |SCALAR +ASCII |SCALAR BIT_LENGTH |SCALAR CHAR |SCALAR CHARACTER_LENGTH |SCALAR diff --git a/x-pack/plugin/sql/qa/src/main/resources/conditionals.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/conditionals.csv-spec index aaa29e814d4b3..9f424da2710e3 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/conditionals.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/conditionals.csv-spec @@ -69,6 +69,19 @@ FROM test_emp ORDER BY 1 LIMIT 5; 10005 | 1 | 10005 ; +caseWithErroneousResultsForFalseConditions +schema::bytes_in:i|bytes_out:i|div:i +SELECT bytes_in, bytes_out, CASE WHEN bytes_in = 0 THEN NULL WHEN bytes_in < 10 THEN bytes_in * 20 ELSE bytes_out / bytes_in END div FROM logs ORDER BY bytes_in LIMIT 5; + + bytes_in | bytes_out | div +---------------+---------------+--------------- +0 |128 |null +0 |null |null +8 |null |160 +8 |null |160 +8 |null |160 +; + caseWhere SELECT last_name FROM test_emp WHERE CASE WHEN LENGTH(last_name) < 7 THEN 'ShortName' ELSE 'LongName' END = 'LongName' ORDER BY emp_no LIMIT 10; @@ -265,6 +278,19 @@ SELECT emp_no, IIF(NULL, emp_no) AS IIF_1, IIF(NULL, emp_no, languages) AS IIF_2 10005 | null | 1 | 10005 ; +iifWithErroneousResultsForFalseCondition +schema::bytes_in:i|bytes_out:i|div:i +SELECT bytes_in, bytes_out, IIF(bytes_in < 10, IIF(bytes_in = 0, NULL, bytes_in * 10), bytes_out / bytes_in) div FROM logs ORDER BY bytes_in LIMIT 5; + + bytes_in | bytes_out | div +---------------+---------------+--------------- +0 |128 |null +0 |null |null +8 |null |80 +8 |null |80 +8 |null |80 +; + iifWhere SELECT last_name FROM test_emp WHERE IIF(LENGTH(last_name) < 7, 'ShortName') IS NOT NULL ORDER BY emp_no LIMIT 10; diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec index 203e707380e19..e7ee913e142c4 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec @@ -318,8 +318,9 @@ SIN |SCALAR SINH |SCALAR SQRT |SCALAR TAN |SCALAR -TRUNCATE |SCALAR -ASCII |SCALAR +TRUNC |SCALAR +TRUNCATE |SCALAR +ASCII |SCALAR BIT_LENGTH |SCALAR CHAR |SCALAR CHARACTER_LENGTH |SCALAR @@ -2032,7 +2033,7 @@ SELECT TRUNCATE(-345.153, -1) AS trimmed; mathTruncateWithPositiveParameter // tag::mathTruncateWithPositiveParameter -SELECT TRUNCATE(-345.153, 1) AS trimmed; +SELECT TRUNC(-345.153, 1) AS trimmed; trimmed --------------- diff --git a/x-pack/plugin/sql/qa/src/main/resources/logs.csv b/x-pack/plugin/sql/qa/src/main/resources/logs.csv index 240fb3752ab53..7103f578b80b6 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/logs.csv +++ b/x-pack/plugin/sql/qa/src/main/resources/logs.csv @@ -25,7 +25,7 @@ id,@timestamp,bytes_in,bytes_out,client_ip,client_port,dest_ip,status 24,2017-11-10T20:34:43Z,8,,10.0.1.166,,2001:cafe::13e1:16fc:8726:1bf8,OK 25,2017-11-10T23:30:46Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK 26,2017-11-10T21:13:16Z,20,,,,2001:cafe::ff07:bdcc:bc59:ff9f,OK -27,2017-11-10T23:36:32Z,0,,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK +27,2017-11-10T23:36:32Z,0,128,10.0.1.199,,2001:cafe::13e1:16fc:8726:1bf8,OK 28,2017-11-10T23:36:33Z,40,,10.0.1.199,,2001:cafe::ff07:bdcc:bc59:ff9f,OK 29,2017-11-10T20:35:26Z,20,,10.0.1.166,,2001:cafe::ff07:bdcc:bc59:ff9f,OK 30,2017-11-10T23:36:41Z,8,,,,2001:cafe::13e1:16fc:8726:1bf8,OK diff --git a/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec index 970724dabd26d..372614dcb0052 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/math.csv-spec @@ -2,9 +2,9 @@ // this one doesn't work in H2 at all truncateWithAsciiHavingAndOrderBy -SELECT TRUNCATE(ASCII(LEFT(first_name, 1)), 1), COUNT(*) count FROM test_emp GROUP BY ASCII(LEFT(first_name, 1)) HAVING COUNT(*) > 5 ORDER BY TRUNCATE(ASCII(LEFT(first_name, 1)), 1) DESC; +SELECT TRUNC(ASCII(LEFT(first_name, 1)), 1), COUNT(*) count FROM test_emp GROUP BY ASCII(LEFT(first_name, 1)) HAVING COUNT(*) > 5 ORDER BY TRUNCATE(ASCII(LEFT(first_name, 1)), 1) DESC; -TRUNCATE(ASCII(LEFT(first_name, 1)), 1):i| count:l +TRUNC(ASCII(LEFT(first_name, 1)), 1):i | count:l -----------------------------------------+--------------- null |10 66 |7 @@ -45,7 +45,7 @@ SELECT ROUND(salary, 2) ROUNDED, salary FROM test_emp GROUP BY ROUNDED, salary O ; truncateWithGroupByAndOrderBy -SELECT TRUNCATE(salary, 2) TRUNCATED, salary FROM test_emp GROUP BY TRUNCATED, salary ORDER BY TRUNCATED LIMIT 10; +SELECT TRUNC(salary, 2) TRUNCATED, salary FROM test_emp GROUP BY TRUNCATED, salary ORDER BY TRUNCATED LIMIT 10; TRUNCATED | salary ---------------+--------------- @@ -129,9 +129,9 @@ SELECT MIN(salary) mi, MAX(salary) ma, YEAR(hire_date) year, ROUND(AVG(languages ; groupByAndOrderByTruncateWithPositiveParameter -SELECT TRUNCATE(AVG(salary),2), AVG(salary), COUNT(*) FROM test_emp GROUP BY TRUNCATE(salary, 2) ORDER BY TRUNCATE(salary, 2) DESC LIMIT 10; +SELECT TRUNC(AVG(salary),2), AVG(salary), COUNT(*) FROM test_emp GROUP BY TRUNC(salary, 2) ORDER BY TRUNCATE(salary, 2) DESC LIMIT 10; -TRUNCATE(AVG(salary),2):d| AVG(salary):d | COUNT(*):l +TRUNC(AVG(salary),2):d | AVG(salary):d | COUNT(*):l -------------------------+---------------+--------------- 74999.0 |74999.0 |1 74970.0 |74970.0 |1 diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java index 5e0c0037af370..cff6fd13a0733 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java @@ -242,7 +242,7 @@ private void defineDefaultFunctions() { def(Sinh.class, Sinh::new, "SINH"), def(Sqrt.class, Sqrt::new, "SQRT"), def(Tan.class, Tan::new, "TAN"), - def(Truncate.class, Truncate::new, "TRUNCATE")); + def(Truncate.class, Truncate::new, "TRUNCATE", "TRUNC")); // String addToMap(def(Ascii.class, Ascii::new, "ASCII"), def(BitLength.class, BitLength::new, "BIT_LENGTH"), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CaseProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CaseProcessor.java index 634e83401fe63..269faba3dc9e1 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CaseProcessor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/conditional/CaseProcessor.java @@ -10,7 +10,6 @@ import org.elasticsearch.xpack.sql.expression.gen.processor.Processor; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -40,14 +39,20 @@ public void writeTo(StreamOutput out) throws IOException { @Override public Object process(Object input) { - List objects = new ArrayList<>(processors.size()); - for (Processor processor : processors) { - objects.add(processor.process(input)); + // Check every condition in sequence and if it evaluates to TRUE, + // evaluate and return the result associated with that condition. + for (int i = 0; i < processors.size() - 2; i += 2) { + if (processors.get(i).process(input) == Boolean.TRUE) { + return processors.get(i + 1).process(input); + } } - return apply(objects); + // resort to default value + return processors.get(processors.size() - 1).process(input); } public static Object apply(List objects) { + // Check every condition in sequence and if it evaluates to TRUE, + // evaluate and return the result associated with that condition. for (int i = 0; i < objects.size() - 2; i += 2) { if (objects.get(i) == Boolean.TRUE) { return objects.get(i + 1); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index 6614d98cffbba..4fbcc76ff8220 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -283,10 +283,20 @@ static GroupingContext groupBy(List groupings) { // dates are handled differently because of date histograms if (exp instanceof DateTimeHistogramFunction) { DateTimeHistogramFunction dthf = (DateTimeHistogramFunction) exp; - if (dthf.calendarInterval() != null) { - key = new GroupByDateHistogram(aggId, nameOf(exp), dthf.calendarInterval(), dthf.zoneId()); - } else { - key = new GroupByDateHistogram(aggId, nameOf(exp), dthf.fixedInterval(), dthf.zoneId()); + Expression field = dthf.field(); + if (field instanceof FieldAttribute) { + if (dthf.calendarInterval() != null) { + key = new GroupByDateHistogram(aggId, nameOf(field), dthf.calendarInterval(), dthf.zoneId()); + } else { + key = new GroupByDateHistogram(aggId, nameOf(field), dthf.fixedInterval(), dthf.zoneId()); + } + } else if (field instanceof Function) { + ScriptTemplate script = ((Function) field).asScript(); + if (dthf.calendarInterval() != null) { + key = new GroupByDateHistogram(aggId, script, dthf.calendarInterval(), dthf.zoneId()); + } else { + key = new GroupByDateHistogram(aggId, script, dthf.fixedInterval(), dthf.zoneId()); + } } } // all other scalar functions become a script diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 2b03810b57982..36722e6e1d0f5 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -999,6 +999,22 @@ public void testGroupByYearQueryTranslator() { + "\"calendar_interval\":\"1y\",\"time_zone\":\"Z\"}}}]}}}")); } + public void testGroupByYearAndScalarsQueryTranslator() { + PhysicalPlan p = optimizeAndPlan("SELECT YEAR(CAST(date + INTERVAL 5 months AS DATE)) FROM test GROUP BY 1"); + assertEquals(EsQueryExec.class, p.getClass()); + EsQueryExec eqe = (EsQueryExec) p; + assertEquals(1, eqe.output().size()); + assertEquals("YEAR(CAST(date + INTERVAL 5 months AS DATE))", eqe.output().get(0).qualifiedName()); + assertEquals(DataType.INTEGER, eqe.output().get(0).dataType()); + assertThat(eqe.queryContainer().aggs().asAggBuilder().toString().replaceAll("\\s+", ""), + endsWith("\"date_histogram\":{\"script\":{\"source\":\"InternalSqlScriptUtils.cast(" + + "InternalSqlScriptUtils.add(InternalSqlScriptUtils.docValue(doc,params.v0)," + + "InternalSqlScriptUtils.intervalYearMonth(params.v1,params.v2)),params.v3)\"," + + "\"lang\":\"painless\",\"params\":{\"v0\":\"date\",\"v1\":\"P5M\",\"v2\":\"INTERVAL_MONTH\"," + + "\"v3\":\"DATE\"}},\"missing_bucket\":true,\"value_type\":\"long\",\"order\":\"asc\"," + + "\"calendar_interval\":\"1y\",\"time_zone\":\"Z\"}}}]}}}")); + } + public void testGroupByHistogramWithDate() { LogicalPlan p = plan("SELECT MAX(int) FROM test GROUP BY HISTOGRAM(CAST(date AS DATE), INTERVAL 2 MONTHS)"); assertTrue(p instanceof Aggregate); diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java index 11a379546b310..2c7524e861f9e 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/history/HistoryTemplateTransformMappingsTests.java @@ -12,12 +12,10 @@ import org.elasticsearch.xpack.core.watcher.transport.actions.put.PutWatchRequestBuilder; import org.elasticsearch.xpack.watcher.test.AbstractWatcherIntegrationTestCase; -import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; +import java.util.Optional; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.xpack.watcher.actions.ActionBuilders.loggingAction; @@ -27,7 +25,6 @@ import static org.elasticsearch.xpack.watcher.transform.TransformBuilders.searchTransform; import static org.elasticsearch.xpack.watcher.trigger.TriggerBuilders.schedule; import static org.elasticsearch.xpack.watcher.trigger.schedule.Schedules.interval; -import static org.hamcrest.Matchers.hasItem; public class HistoryTemplateTransformMappingsTests extends AbstractWatcherIntegrationTestCase { @@ -77,19 +74,16 @@ public void testTransformFields() throws Exception { GetFieldMappingsResponse response = client().admin().indices() .prepareGetFieldMappings(".watcher-history*") .setFields("result.actions.transform.payload") - .setTypes(SINGLE_MAPPING_NAME) .includeDefaults(true) .get(); // time might have rolled over to a new day, thus we need to check that this field exists only in one of the history indices - List payloadNulls = response.mappings().values().stream() - .map(map -> map.get(SINGLE_MAPPING_NAME)) + Optional mapping = response.mappings().values().stream() .map(map -> map.get("result.actions.transform.payload")) .filter(Objects::nonNull) - .map(GetFieldMappingsResponse.FieldMappingMetaData::isNull) - .collect(Collectors.toList()); + .findFirst(); - assertThat(payloadNulls, hasItem(true)); + assertTrue(mapping.isEmpty()); }); } } From 32ba24b0710872c4262c349c1a237fa1a8fa5df8 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 28 Nov 2019 15:59:09 +0100 Subject: [PATCH 37/62] Add new encoding strategy based on Geometry tessellation. (#47903) --- .../common/geo/EdgeTreeReader.java | 10 + .../common/geo/GeometryTreeReader.java | 2 + .../common/geo/Point2DReader.java | 14 +- .../common/geo/PolygonTreeReader.java | 10 + .../common/geo/ShapeTreeReader.java | 2 + .../common/geo/TriangleTreeReader.java | 395 +++++++++++ .../common/geo/TriangleTreeWriter.java | 618 ++++++++++++++++++ .../common/geo/AbstractTreeTestCase.java | 360 ++++++++++ .../common/geo/EncodingComparisonTests.java | 207 ++++++ .../common/geo/GeometryTreeTests.java | 334 +--------- .../common/geo/TriangleTreeTests.java | 36 + 11 files changed, 1655 insertions(+), 333 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java create mode 100644 server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index a6292d8d51b41..a67b296398b31 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -54,6 +54,16 @@ public Extent getExtent() throws IOException { return treeExtent; } + @Override + public double getCentroidX() { + throw new UnsupportedOperationException(); + } + + @Override + public double getCentroidY() { + throw new UnsupportedOperationException(); + } + /** * Returns true if the rectangle query and the edge tree's shape overlap */ diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 36c0710de1090..b98c8a19fb5df 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -51,11 +51,13 @@ private GeometryTreeReader(ByteBufferStreamInput input, CoordinateEncoder coordi this.coordinateEncoder = coordinateEncoder; } + @Override public double getCentroidX() throws IOException { input.position(startPosition); return coordinateEncoder.decodeX(input.readInt()); } + @Override public double getCentroidY() throws IOException { input.position(startPosition + 4); return coordinateEncoder.decodeY(input.readInt()); diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java index 1fe9cb39faef8..737386026a09a 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java @@ -39,6 +39,7 @@ class Point2DReader implements ShapeTreeReader { this.startPosition = input.position(); } + @Override public Extent getExtent() throws IOException { if (size == 1) { int x = readX(0); @@ -50,6 +51,17 @@ public Extent getExtent() throws IOException { } } + @Override + public double getCentroidX() { + throw new UnsupportedOperationException(); + } + + @Override + public double getCentroidY() { + throw new UnsupportedOperationException(); + } + + @Override public GeoRelation relate(Extent extent) throws IOException { Deque stack = new ArrayDeque<>(); @@ -73,7 +85,7 @@ public GeoRelation relate(Extent extent) throws IOException { continue; } - int middle = (right - left) >> 1; + int middle = (right + left) >> 1; int x = readX(middle); int y = readY(middle); if (x >= extent.minX() && x <= extent.maxX() && y >= extent.minY() && y <= extent.maxY()) { diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java index 663e771cde67d..6474a17da5f0b 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java @@ -47,6 +47,16 @@ public Extent getExtent() throws IOException { return outerShell.getExtent(); } + @Override + public double getCentroidX() { + throw new UnsupportedOperationException(); + } + + @Override + public double getCentroidY() { + throw new UnsupportedOperationException(); + } + /** * Returns true if the rectangle query and the edge tree's shape overlap */ diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java index 0d5fb2a6d6b2d..9bb88980e2f0c 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java @@ -29,4 +29,6 @@ public interface ShapeTreeReader { Extent getExtent() throws IOException; GeoRelation relate(Extent extent) throws IOException; + double getCentroidX() throws IOException; + double getCentroidY() throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java new file mode 100644 index 0000000000000..a39300f969d84 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -0,0 +1,395 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import static org.apache.lucene.geo.GeoUtils.orient; + +/** + * A tree reader for a previous serialized {@link org.elasticsearch.geometry.Geometry} using + * {@link TriangleTreeWriter}. + * + * This class supports checking bounding box + * relations against the serialized triangle tree. + */ +public class TriangleTreeReader implements ShapeTreeReader { + + private final int extentOffset = 8; + private final ByteBufferStreamInput input; + private final CoordinateEncoder coordinateEncoder; + private final Rectangle2D rectangle2D; + + public TriangleTreeReader(BytesRef bytesRef, CoordinateEncoder coordinateEncoder) { + this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + this.coordinateEncoder = coordinateEncoder; + this.rectangle2D = new Rectangle2D(); + } + + /** + * returns the bounding box of the geometry in the format [minX, maxX, minY, maxY]. + */ + public Extent getExtent() throws IOException { + input.position(extentOffset); + int thisMaxX = input.readInt(); + int thisMinX = Math.toIntExact(thisMaxX - input.readVLong()); + int thisMaxY = input.readInt(); + int thisMinY = Math.toIntExact(thisMaxY - input.readVLong()); + return Extent.fromPoints(thisMinX, thisMinY, thisMaxX, thisMaxY); + } + + /** + * returns the X coordinate of the centroid. + */ + @Override + public double getCentroidX() throws IOException { + input.position(0); + return coordinateEncoder.decodeX(input.readInt()); + } + + /** + * returns the Y coordinate of the centroid. + */ + @Override + public double getCentroidY() throws IOException { + input.position(4); + return coordinateEncoder.decodeY(input.readInt()); + } + + @Override + public GeoRelation relate(Extent extent) throws IOException { + return relate(extent.minX(), extent.maxX(), extent.minY(), extent.maxY()); + } + + /** + * Compute the relation with the provided bounding box. If the result is CELL_INSIDE_QUERY + * then the bounding box is within the shape. + */ + private GeoRelation relate(int minX, int maxX, int minY, int maxY) throws IOException { + input.position(extentOffset); + int thisMaxX = input.readInt(); + int thisMinX = Math.toIntExact(thisMaxX - input.readVLong()); + int thisMaxY = input.readInt(); + int thisMinY = Math.toIntExact(thisMaxY - input.readVLong()); + if (minX <= thisMinX && maxX >= thisMaxX && minY <= thisMinY && maxY >= thisMaxY) { + // the rectangle fully contains the shape + return GeoRelation.QUERY_CROSSES; + } + GeoRelation rel = GeoRelation.QUERY_DISJOINT; + if ((thisMinX > maxX || thisMaxX < minX || thisMinY > maxY || thisMaxY < minY) == false) { + // shapes are NOT disjoint + rectangle2D.setValues(minX, maxX, minY, maxY); + byte metadata = input.readByte(); + if ((metadata & 1 << 2) == 1 << 2) { // component in this node is a point + int x = Math.toIntExact(thisMaxX - input.readVLong()); + int y = Math.toIntExact(thisMaxY - input.readVLong()); + if (rectangle2D.contains(x, y)) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = x; + } else if ((metadata & 1 << 3) == 1 << 3) { // component in this node is a line + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + if (rectangle2D.intersectsLine(aX, aY, bX, bY)) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + } else { // component in this node is a triangle + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + int cX = Math.toIntExact(thisMaxX - input.readVLong()); + int cY = Math.toIntExact(thisMaxY - input.readVLong()); + boolean ab = (metadata & 1 << 4) == 1 << 4; + boolean bc = (metadata & 1 << 5) == 1 << 5; + boolean ca = (metadata & 1 << 6) == 1 << 6; + rel = rectangle2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); + if (rel == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + } + if ((metadata & 1 << 0) == 1 << 0) { // left != null + GeoRelation left = relate(rectangle2D, false, thisMaxX, thisMaxY); + if (left == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (left == GeoRelation.QUERY_INSIDE) { + rel = left; + } + } + if ((metadata & 1 << 1) == 1 << 1) { // right != null + if (rectangle2D.maxX >= thisMinX) { + GeoRelation right = relate(rectangle2D, false, thisMaxX, thisMaxY); + if (right == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (right == GeoRelation.QUERY_INSIDE) { + rel = right; + } + } + } + } + return rel; + } + + private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMaxX, int parentMaxY) throws IOException { + int thisMaxX = Math.toIntExact(parentMaxX - input.readVLong()); + int thisMaxY = Math.toIntExact(parentMaxY - input.readVLong()); + GeoRelation rel = GeoRelation.QUERY_DISJOINT; + int size = input.readVInt(); + if (rectangle2D.minY <= thisMaxY && rectangle2D.minX <= thisMaxX) { + byte metadata = input.readByte(); + int thisMinX; + int thisMinY; + if ((metadata & 1 << 2) == 1 << 2) { // component in this node is a point + int x = Math.toIntExact(thisMaxX - input.readVLong()); + int y = Math.toIntExact(thisMaxY - input.readVLong()); + if (rectangle2D.contains(x, y)) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = x; + thisMinY = y; + } else if ((metadata & 1 << 3) == 1 << 3) { // component in this node is a line + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + if (rectangle2D.intersectsLine(aX, aY, bX, bY)) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + thisMinY = Math.min(aY, bY); + } else { // component in this node is a triangle + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + int cX = Math.toIntExact(thisMaxX - input.readVLong()); + int cY = Math.toIntExact(thisMaxY - input.readVLong()); + boolean ab = (metadata & 1 << 4) == 1 << 4; + boolean bc = (metadata & 1 << 5) == 1 << 5; + boolean ca = (metadata & 1 << 6) == 1 << 6; + rel = rectangle2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); + if (rel == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + thisMinY = Math.min(Math.min(aY, bY), cY); + } + if ((metadata & 1 << 0) == 1 << 0) { // left != null + GeoRelation left = relate(rectangle2D, !splitX, thisMaxX, thisMaxY); + if (left == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (left == GeoRelation.QUERY_INSIDE) { + rel = left; + } + } + if ((metadata & 1 << 1) == 1 << 1) { // right != null + int rightSize = input.readVInt(); + if ((splitX == false && rectangle2D.maxY >= thisMinY) || (splitX && rectangle2D.maxX >= thisMinX)) { + GeoRelation right = relate(rectangle2D, !splitX, thisMaxX, thisMaxY); + if (right == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (right == GeoRelation.QUERY_INSIDE) { + rel = right; + } + } else { + input.skip(rightSize); + } + } + } else { + input.skip(size); + } + return rel; + } + + private static class Rectangle2D { + + protected int minX; + protected int maxX; + protected int minY; + protected int maxY; + + Rectangle2D() { + } + + protected void setValues(int minX, int maxX, int minY, int maxY) { + this.minX = minX; + this.maxX = maxX; + this.minY = minY; + this.maxY = maxY; + } + + /** + * Checks if the rectangle contains the provided point + **/ + public boolean contains(int x, int y) { + return (x < minX || x > maxX || y < minY || y > maxY) == false; + } + + /** + * Checks if the rectangle intersects the provided triangle + **/ + public boolean intersectsLine(int aX, int aY, int bX, int bY) { + // 1. query contains any triangle points + if (contains(aX, aY) || contains(bX, bY)) { + return true; + } + + // compute bounding box of triangle + int tMinX = StrictMath.min(aX, bX); + int tMaxX = StrictMath.max(aX, bX); + int tMinY = StrictMath.min(aY, bY); + int tMaxY = StrictMath.max(aY, bY); + + // 2. check bounding boxes are disjoint + if (tMaxX < minX || tMinX > maxX || tMinY > maxY || tMaxY < minY) { + return false; + } + + // 4. last ditch effort: check crossings + if (edgeIntersectsQuery(aX, aY, bX, bY)) { + return true; + } + return false; + } + + /** + * Checks if the rectangle intersects the provided triangle + **/ + public GeoRelation relateTriangle(int aX, int aY, boolean ab, int bX, int bY, boolean bc, int cX, int cY, boolean ca) { + // 1. query contains any triangle points + if (contains(aX, aY) || contains(bX, bY) || contains(cX, cY)) { + return GeoRelation.QUERY_CROSSES; + } + + // compute bounding box of triangle + int tMinX = StrictMath.min(StrictMath.min(aX, bX), cX); + int tMaxX = StrictMath.max(StrictMath.max(aX, bX), cX); + int tMinY = StrictMath.min(StrictMath.min(aY, bY), cY); + int tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY); + + // 2. check bounding boxes are disjoint + if (tMaxX < minX || tMinX > maxX || tMinY > maxY || tMaxY < minY) { + return GeoRelation.QUERY_DISJOINT; + } + + boolean within = false; + if (edgeIntersectsQuery(aX, aY, bX, bY)) { + if (ab) { + return GeoRelation.QUERY_CROSSES; + } + within = true; + } + + // right + if (edgeIntersectsQuery(bX, bY, cX, cY)) { + if (bc) { + return GeoRelation.QUERY_CROSSES; + } + within = true; + } + + if (edgeIntersectsQuery(cX, cY, aX, aY)) { + if (ca) { + return GeoRelation.QUERY_CROSSES; + } + within = true; + } + + if (within || pointInTriangle(tMinX, tMaxX, tMinY, tMaxY, minX, minY, aX, aY, bX, bY, cX, cY)) { + return GeoRelation.QUERY_INSIDE; + } + + return GeoRelation.QUERY_DISJOINT; + } + + /** + * returns true if the edge (defined by (ax, ay) (bx, by)) intersects the query + */ + private boolean edgeIntersectsQuery(int ax, int ay, int bx, int by) { + // shortcut: check bboxes of edges are disjoint + if (boxesAreDisjoint(Math.min(ax, bx), Math.max(ax, bx), Math.min(ay, by), Math.max(ay, by), + minX, maxX, minY, maxY)) { + return false; + } + + // top + if (orient(ax, ay, bx, by, minX, maxY) * orient(ax, ay, bx, by, maxX, maxY) <= 0 && + orient(minX, maxY, maxX, maxY, ax, ay) * orient(minX, maxY, maxX, maxY, bx, by) <= 0) { + return true; + } + + // right + if (orient(ax, ay, bx, by, maxX, maxY) * orient(ax, ay, bx, by, maxX, minY) <= 0 && + orient(maxX, maxY, maxX, minY, ax, ay) * orient(maxX, maxY, maxX, minY, bx, by) <= 0) { + return true; + } + + // bottom + if (orient(ax, ay, bx, by, maxX, minY) * orient(ax, ay, bx, by, minX, minY) <= 0 && + orient(maxX, minY, minX, minY, ax, ay) * orient(maxX, minY, minX, minY, bx, by) <= 0) { + return true; + } + + // left + if (orient(ax, ay, bx, by, minX, minY) * orient(ax, ay, bx, by, minX, maxY) <= 0 && + orient(minX, minY, minX, maxY, ax, ay) * orient(minX, minY, minX, maxY, bx, by) <= 0) { + return true; + } + + return false; + } + + /** + * Compute whether the given x, y point is in a triangle; uses the winding order method + */ + static boolean pointInTriangle(double minX, double maxX, double minY, double maxY, double x, double y, + double aX, double aY, double bX, double bY, double cX, double cY) { + //check the bounding box because if the triangle is degenerated, e.g points and lines, we need to filter out + //coplanar points that are not part of the triangle. + if (x >= minX && x <= maxX && y >= minY && y <= maxY) { + int a = orient(x, y, aX, aY, bX, bY); + int b = orient(x, y, bX, bY, cX, cY); + if (a == 0 || b == 0 || a < 0 == b < 0) { + int c = orient(x, y, cX, cY, aX, aY); + return c == 0 || (c < 0 == (b < 0 || a < 0)); + } + return false; + } else { + return false; + } + } + + /** + * utility method to check if two boxes are disjoint + */ + private static boolean boxesAreDisjoint(final int aMinX, final int aMaxX, final int aMinY, final int aMaxY, + final int bMinX, final int bMaxX, final int bMinY, final int bMaxY) { + return (aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY); + } + + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java new file mode 100644 index 0000000000000..95e58bc056c7e --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java @@ -0,0 +1,618 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.geo.GeoUtils; +import org.apache.lucene.geo.Tessellator; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.index.mapper.GeoShapeIndexer; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +/** + * This is a tree-writer that serializes a {@link Geometry} and tessellate it to write it into a byte array. + * Internally it tessellate the given {@link Geometry} and it builds an interval tree with the + * tessellation. + */ +public class TriangleTreeWriter extends ShapeTreeWriter { + + private final TriangleTreeNode node; + private final CoordinateEncoder coordinateEncoder; + private final CentroidCalculator centroidCalculator; + private final ShapeType type; + + public TriangleTreeWriter(Geometry geometry, CoordinateEncoder coordinateEncoder) { + this.coordinateEncoder = coordinateEncoder; + this.centroidCalculator = new CentroidCalculator(); + this.type = geometry.type(); + TriangleTreeBuilder builder = new TriangleTreeBuilder(coordinateEncoder); + geometry.visit(builder); + this.node = builder.build(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); + out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); + node.writeTo(out); + } + + @Override + public Extent getExtent() { + // do it right + return Extent.fromPoints(node.minX, node.minY, node.maxX, node.maxY); + } + + @Override + public ShapeType getShapeType() { + return type; + } + + @Override + public CentroidCalculator getCentroidCalculator() { + return centroidCalculator; + } + + /** + * Class that tessellate the geometry and build an interval tree in memory. + */ + class TriangleTreeBuilder implements GeometryVisitor { + + private List triangles; + private final CoordinateEncoder coordinateEncoder; + + TriangleTreeBuilder(CoordinateEncoder coordinateEncoder) { + this.coordinateEncoder = coordinateEncoder; + this.triangles = new ArrayList<>(); + } + + private void addTriangles(List triangles) { + this.triangles.addAll(triangles); + } + + @Override + public Void visit(GeometryCollection collection) { + for (Geometry geometry : collection) { + geometry.visit(this); + } + return null; + } + + @Override + public Void visit(Line line) { + for (int i =0; i < line.length(); i++) { + centroidCalculator.addCoordinate(line.getX(i), line.getY(i)); + } + addTriangles(TriangleTreeLeaf.fromLine(coordinateEncoder, line)); + return null; + } + + @Override + public Void visit(MultiLine multiLine) { + for (Line line : multiLine) { + visit(line); + } + return null; + } + + @Override + public Void visit(Polygon polygon) { + // TODO: Shall we consider holes for centroid computation? + for (int i =0; i < polygon.getPolygon().length() - 1; i++) { + centroidCalculator.addCoordinate(polygon.getPolygon().getX(i), polygon.getPolygon().getY(i)); + } + addTriangles(TriangleTreeLeaf.fromPolygon(coordinateEncoder, polygon)); + return null; + } + + @Override + public Void visit(MultiPolygon multiPolygon) { + for (Polygon polygon : multiPolygon) { + visit(polygon); + } + return null; + } + + @Override + public Void visit(Rectangle r) { + centroidCalculator.addCoordinate(r.getMinX(), r.getMinY()); + centroidCalculator.addCoordinate(r.getMaxX(), r.getMaxY()); + addTriangles(TriangleTreeLeaf.fromRectangle(coordinateEncoder, r)); + return null; + } + + @Override + public Void visit(Point point) { + centroidCalculator.addCoordinate(point.getX(), point.getY()); + addTriangles(TriangleTreeLeaf.fromPoints(coordinateEncoder, point)); + return null; + } + + @Override + public Void visit(MultiPoint multiPoint) { + for (Point point : multiPoint) { + visit(point); + } + return null; + } + + @Override + public Void visit(LinearRing ring) { + throw new IllegalArgumentException("invalid shape type found [LinearRing]"); + } + + @Override + public Void visit(Circle circle) { + throw new IllegalArgumentException("invalid shape type found [Circle]"); + } + + + public TriangleTreeNode build() { + if (triangles.size() == 1) { + return new TriangleTreeNode(triangles.get(0)); + } + TriangleTreeNode[] nodes = new TriangleTreeNode[triangles.size()]; + for (int i = 0; i < triangles.size(); i++) { + nodes[i] = new TriangleTreeNode(triangles.get(i)); + } + TriangleTreeNode root = createTree(nodes, 0, triangles.size() - 1, true); + for (TriangleTreeNode node : nodes) { + root.minX = Math.min(root.minX, node.minX); + root.minY = Math.min(root.minY, node.minY); + } + return root; + } + + /** Creates tree from sorted components (with range low and high inclusive) */ + private TriangleTreeNode createTree(TriangleTreeNode[] components, int low, int high, boolean splitX) { + if (low > high) { + return null; + } + final int mid = (low + high) >>> 1; + if (low < high) { + Comparator comparator; + if (splitX) { + comparator = (left, right) -> { + int ret = Double.compare(left.minX, right.minX); + if (ret == 0) { + ret = Double.compare(left.maxX, right.maxX); + } + return ret; + }; + } else { + comparator = (left, right) -> { + int ret = Double.compare(left.minY, right.minY); + if (ret == 0) { + ret = Double.compare(left.maxY, right.maxY); + } + return ret; + }; + } + ArrayUtil.select(components, low, high + 1, mid, comparator); + } + TriangleTreeNode newNode = components[mid]; + // find children + newNode.left = createTree(components, low, mid - 1, !splitX); + newNode.right = createTree(components, mid + 1, high, !splitX); + + // pull up max values to this node + if (newNode.left != null) { + newNode.maxX = Math.max(newNode.maxX, newNode.left.maxX); + newNode.maxY = Math.max(newNode.maxY, newNode.left.maxY); + } + if (newNode.right != null) { + newNode.maxX = Math.max(newNode.maxX, newNode.right.maxX); + newNode.maxY = Math.max(newNode.maxY, newNode.right.maxY); + } + return newNode; + } + } + + /** + * Represents an inner node of the tree. + */ + static class TriangleTreeNode implements Writeable { + /** minimum latitude of this geometry's bounding box area */ + private int minY; + /** maximum latitude of this geometry's bounding box area */ + private int maxY; + /** minimum longitude of this geometry's bounding box area */ + private int minX; + /** maximum longitude of this geometry's bounding box area */ + private int maxX; + // child components, or null. Note internal nodes might mot have + // a consistent bounding box. Internal nodes should not be accessed + // outside if this class. + private TriangleTreeNode left; + private TriangleTreeNode right; + /** root node of edge tree */ + private TriangleTreeLeaf component; + + protected TriangleTreeNode(TriangleTreeLeaf component) { + this.minY = component.minY; + this.maxY = component.maxY; + this.minX = component.minX; + this.maxX = component.maxX; + this.component = component; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + BytesStreamOutput scratchBuffer = new BytesStreamOutput(); + out.writeInt(maxX); + out.writeVLong((long) maxX - minX); + out.writeInt(maxY); + out.writeVLong((long) maxY - minY); + writeMetadata(out); + writeComponent(out); + if (left != null) { + left.writeNode(out, maxX, maxY, scratchBuffer); + } + if (right != null) { + right.writeNode(out, maxX, maxY, scratchBuffer); + } + } + + private void writeNode(StreamOutput out, int parentMaxX, int parentMaxY, BytesStreamOutput scratchBuffer) throws IOException { + out.writeVLong((long) parentMaxX - maxX); + out.writeVLong((long) parentMaxY - maxY); + int size = nodeSize(false, parentMaxX, parentMaxY, scratchBuffer); + out.writeVInt(size); + writeMetadata(out); + writeComponent(out); + if (left != null) { + left.writeNode(out, maxX, maxY, scratchBuffer); + } + if (right != null) { + int rightSize = right.nodeSize(true, maxX, maxY,scratchBuffer); + out.writeVInt(rightSize); + right.writeNode(out, maxX, maxY, scratchBuffer); + } + } + + private void writeMetadata(StreamOutput out) throws IOException { + byte metadata = 0; + metadata |= (left != null) ? (1 << 0) : 0; + metadata |= (right != null) ? (1 << 1) : 0; + if (component.type == TriangleTreeLeaf.TYPE.POINT) { + metadata |= (1 << 2); + } else if (component.type == TriangleTreeLeaf.TYPE.LINE) { + metadata |= (1 << 3); + } else { + metadata |= (component.ab) ? (1 << 4) : 0; + metadata |= (component.bc) ? (1 << 5) : 0; + metadata |= (component.ca) ? (1 << 6) : 0; + } + out.writeByte(metadata); + } + + private void writeComponent(StreamOutput out) throws IOException { + if (component.type == TriangleTreeLeaf.TYPE.POINT) { + out.writeVLong((long) maxX - component.aX); + out.writeVLong((long) maxY - component.aY); + } else if (component.type == TriangleTreeLeaf.TYPE.LINE) { + out.writeVLong((long) maxX - component.aX); + out.writeVLong((long) maxY - component.aY); + out.writeVLong((long) maxX - component.bX); + out.writeVLong((long) maxY - component.bY); + } else { + out.writeVLong((long) maxX - component.aX); + out.writeVLong((long) maxY - component.aY); + out.writeVLong((long) maxX - component.bX); + out.writeVLong((long) maxY - component.bY); + out.writeVLong((long) maxX - component.cX); + out.writeVLong((long) maxY - component.cY); + } + } + + public int nodeSize(boolean includeBox, int parentMaxX, int parentMaxY, BytesStreamOutput scratchBuffer) throws IOException { + int size =0; + size++; //metadata + size += componentSize(scratchBuffer); + if (left != null) { + size += left.nodeSize(true, maxX, maxY, scratchBuffer); + } + if (right != null) { + int rightSize = right.nodeSize(true, maxX, maxY, scratchBuffer); + scratchBuffer.reset(); + scratchBuffer.writeVLong(rightSize); + size += scratchBuffer.size(); // jump size + size += rightSize; + } + if (includeBox) { + int jumpSize = size; + scratchBuffer.reset(); + scratchBuffer.writeVLong((long) parentMaxX - maxX); + scratchBuffer.writeVLong((long) parentMaxY - maxY); + scratchBuffer.writeVLong(jumpSize); + size += scratchBuffer.size();// box + } + return size; + } + + public int componentSize(BytesStreamOutput scratchBuffer) throws IOException { + scratchBuffer.reset(); + if (component.type == TriangleTreeLeaf.TYPE.POINT) { + scratchBuffer.writeVLong((long) maxX - component.aX); + scratchBuffer.writeVLong((long) maxY - component.aY); + } else if (component.type == TriangleTreeLeaf.TYPE.LINE) { + scratchBuffer.writeVLong((long) maxX - component.aX); + scratchBuffer.writeVLong((long) maxY - component.aY); + scratchBuffer.writeVLong((long) maxX - component.bX); + scratchBuffer.writeVLong((long) maxY - component.bY); + } else { + scratchBuffer.writeVLong((long) maxX - component.aX); + scratchBuffer.writeVLong((long) maxY - component.aY); + scratchBuffer.writeVLong((long) maxX - component.bX); + scratchBuffer.writeVLong((long) maxY - component.bY); + scratchBuffer.writeVLong((long) maxX - component.cX); + scratchBuffer.writeVLong((long) maxY - component.cY); + } + return scratchBuffer.size(); + } + + } + + /** + * Represents an leaf of the tree containing one of the triangles. + */ + static class TriangleTreeLeaf { + + public enum TYPE { + POINT, LINE, TRIANGLE + } + + int minX, maxX, minY, maxY; + int aX, aY, bX, bY, cX, cY; + boolean ab, bc, ca; + TYPE type; + + // constructor for points + TriangleTreeLeaf(int aXencoded, int aYencoded) { + encodePoint(aXencoded, aYencoded); + } + + // constructor for points and lines + TriangleTreeLeaf(int aXencoded, int aYencoded, int bXencoded, int bYencoded) { + if (aXencoded == bXencoded && aYencoded == bYencoded) { + encodePoint(aXencoded, aYencoded); + } else { + encodeLine(aXencoded, aYencoded, bXencoded, bYencoded); + } + } + + // generic constructor + TriangleTreeLeaf(int aXencoded, int aYencoded, boolean ab, + int bXencoded, int bYencoded, boolean bc, + int cXencoded, int cYencoded, boolean ca) { + if (aXencoded == bXencoded && aYencoded == bYencoded) { + if (aXencoded == cXencoded && aYencoded == cYencoded) { + encodePoint(aYencoded, aXencoded); + } else { + encodeLine(aYencoded, aXencoded, cYencoded, cXencoded); + return; + } + } else if (aXencoded == cXencoded && aYencoded == cYencoded) { + encodeLine(aYencoded, aXencoded, bYencoded, bXencoded); + } else { + encodeTriangle(aXencoded, aYencoded, ab, bXencoded, bYencoded, bc, cXencoded, cYencoded, ca); + } + } + + private void encodePoint(int aXencoded, int aYencoded) { + this.type = TYPE.POINT; + aX = aXencoded; + aY = aYencoded; + minX = aX; + maxX = aX; + minY = aY; + maxY = aY; + } + + private void encodeLine(int aXencoded, int aYencoded, int bXencoded, int bYencoded) { + this.type = TYPE.LINE; + //rotate edges and place minX at the beginning + if (aXencoded > bXencoded) { + aX = bXencoded; + aY = bYencoded; + bX = aXencoded; + bY = aYencoded; + } else { + aX = aXencoded; + aY = aYencoded; + bX = bXencoded; + bY = bYencoded; + } + this.minX = aX; + this.maxX = bX; + this.minY = Math.min(aY, bY); + this.maxY = Math.max(aY, bY); + } + + private void encodeTriangle(int aXencoded, int aYencoded, boolean abFromShape, + int bXencoded, int bYencoded, boolean bcFromShape, + int cXencoded, int cYencoded, boolean caFromShape) { + + int aX, aY, bX, bY, cX, cY; + boolean ab, bc, ca; + //change orientation if CW + if (GeoUtils.orient(aXencoded, aYencoded, bXencoded, bYencoded, cXencoded, cYencoded) == -1) { + aX = cXencoded; + bX = bXencoded; + cX = aXencoded; + aY = cYencoded; + bY = bYencoded; + cY = aYencoded; + ab = bcFromShape; + bc = abFromShape; + ca = caFromShape; + } else { + aX = aXencoded; + bX = bXencoded; + cX = cXencoded; + aY = aYencoded; + bY = bYencoded; + cY = cYencoded; + ab = abFromShape; + bc = bcFromShape; + ca = caFromShape; + } + //rotate edges and place minX at the beginning + if (bX < aX || cX < aX) { + if (bX < cX) { + int tempX = aX; + int tempY = aY; + boolean tempBool = ab; + aX = bX; + aY = bY; + ab = bc; + bX = cX; + bY = cY; + bc = ca; + cX = tempX; + cY = tempY; + ca = tempBool; + } else if (cX < aX) { + int tempX = aX; + int tempY = aY; + boolean tempBool = ab; + aX = cX; + aY = cY; + ab = ca; + cX = bX; + cY = bY; + ca = bc; + bX = tempX; + bY = tempY; + bc = tempBool; + } + } else if (aX == bX && aX == cX) { + //degenerated case, all points with same longitude + //we need to prevent that aX is in the middle (not part of the MBS) + if (bY < aY || cY < aY) { + if (bY < cY) { + int tempX = aX; + int tempY = aY; + boolean tempBool = ab; + aX = bX; + aY = bY; + ab = bc; + bX = cX; + bY = cY; + bc = ca; + cX = tempX; + cY = tempY; + ca = tempBool; + } else if (cY < aY) { + int tempX = aX; + int tempY = aY; + boolean tempBool = ab; + aX = cX; + aY = cY; + ab = ca; + cX = bX; + cY = bY; + ca = bc; + bX = tempX; + bY = tempY; + bc = tempBool; + } + } + } + this.aX = aX; + this.aY = aY; + this.bX = bX; + this.bY = bY; + this.cX = cX; + this.cY = cY; + this.ab = ab; + this.bc = bc; + this.ca = ca; + this.minX = aX; + this.maxX = Math.max(aX, Math.max(bX, cX)); + this.minY = Math.min(aY, Math.min(bY, cY)); + this.maxY = Math.max(aY, Math.max(bY, cY)); + type = TYPE.TRIANGLE; + } + + private static List fromPoints(CoordinateEncoder encoder, Point... points) { + List triangles = new ArrayList<>(points.length); + for (int i = 0; i < points.length; i++) { + triangles.add(new TriangleTreeLeaf(encoder.encodeX(points[i].getX()), encoder.encodeY(points[i].getY()))); + } + return triangles; + } + + private static List fromRectangle(CoordinateEncoder encoder, Rectangle... rectangles) { + List triangles = new ArrayList<>(2 * rectangles.length); + for (Rectangle r : rectangles) { + triangles.add(new TriangleTreeLeaf( + encoder.encodeX(r.getMinX()), encoder.encodeY(r.getMinY()), true, + encoder.encodeX(r.getMaxX()), encoder.encodeY(r.getMinY()), false, + encoder.encodeX(r.getMinX()), encoder.encodeY(r.getMaxY()), true)); + triangles.add(new TriangleTreeLeaf( + encoder.encodeX(r.getMinX()), encoder.encodeY(r.getMaxY()), false, + encoder.encodeX(r.getMaxX()), encoder.encodeY(r.getMinY()), true, + encoder.encodeX(r.getMaxX()), encoder.encodeY(r.getMaxY()), true)); + } + return triangles; + } + + private static List fromLine(CoordinateEncoder encoder, Line line) { + List triangles = new ArrayList<>(line.length() - 1); + for (int i = 0, j = 1; i < line.length() - 1; i++, j++) { + triangles.add(new TriangleTreeLeaf(encoder.encodeX(line.getX(i)), encoder.encodeY(line.getY(i)), + encoder.encodeX(line.getX(j)), encoder.encodeY(line.getY(j)))); + } + return triangles; + } + + private static List fromPolygon(CoordinateEncoder encoder, Polygon polygon) { + // TODO: We are going to be tessellating the polygon twice, can we do something? + // TODO: Tessellator seems to have some reference to the encoding but does not need to have. + List tessellation = Tessellator.tessellate(GeoShapeIndexer.toLucenePolygon(polygon)); + List triangles = new ArrayList<>(tessellation.size()); + for (Tessellator.Triangle t : tessellation) { + triangles.add(new TriangleTreeLeaf(encoder.encodeX(t.getX(0)), encoder.encodeY(t.getY(0)), t.isEdgefromPolygon(0), + encoder.encodeX(t.getX(1)), encoder.encodeY(t.getY(1)), t.isEdgefromPolygon(1), + encoder.encodeX(t.getX(2)), encoder.encodeY(t.getY(2)), t.isEdgefromPolygon(2))); + } + return triangles; + } + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java b/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java new file mode 100644 index 0000000000000..7e3652fe444c1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java @@ -0,0 +1,360 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.io.stream.ByteBufferStreamInput; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.geo.RandomShapeGenerator; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; +import static org.elasticsearch.geo.GeometryTestUtils.fold; +import static org.elasticsearch.geo.GeometryTestUtils.randomPoint; +import static org.hamcrest.Matchers.equalTo; + +public abstract class AbstractTreeTestCase extends ESTestCase { + + public void testRectangleShape() throws IOException { + for (int i = 0; i < 1000; i++) { + int minX = randomIntBetween(-80, 70); + int maxX = randomIntBetween(minX + 10, 80); + int minY = randomIntBetween(-80, 70); + int maxY = randomIntBetween(minY + 10, 80); + double[] x = new double[]{minX, maxX, maxX, minX, minX}; + double[] y = new double[]{minY, minY, maxY, maxY, minY}; + Geometry rectangle = randomBoolean() ? + new Polygon(new LinearRing(x, y), Collections.emptyList()) : new Rectangle(minX, maxX, maxY, minY); + ShapeTreeReader reader = geometryTreeReader(rectangle, TestCoordinateEncoder.INSTANCE); + + assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); + // encoder loses precision when casting to integer, so centroid is calculated using integer division here + assertThat(reader.getCentroidX(), equalTo((double) ((minX + maxX) / 2))); + assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY) / 2))); + + // box-query touches bottom-left corner + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), + minY - randomIntBetween(1, 180), minX, minY)); + // box-query touches bottom-right corner + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), + maxX + randomIntBetween(1, 180), minY)); + // box-query touches top-right corner + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), + maxY + randomIntBetween(1, 180))); + // box-query touches top-left corner + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, + maxY + randomIntBetween(1, 180))); + // box-query fully-enclosed inside rectangle + assertRelation(GeoRelation.QUERY_INSIDE,reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); + // box-query fully-contains poly + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), + minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + // box-query half-in-half-out-right + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + // box-query half-in-half-out-left + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, + (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); + // box-query half-in-half-out-top + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + maxX + randomIntBetween(1, 1000), maxY + randomIntBetween(1, 1000))); + // box-query half-in-half-out-bottom + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + + // box-query outside to the right + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, + maxX + randomIntBetween(1001, 2000), maxY)); + // box-query outside to the left + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, + minX - randomIntBetween(1, 1000), maxY)); + // box-query outside to the top + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, + maxY + randomIntBetween(1001, 2000))); + // box-query outside to the bottom + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, + minY - randomIntBetween(1, 1000))); + } + } + + public void testSimplePolygon() throws IOException { + for (int iter = 0; iter < 1000; iter++) { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "name"); + ShapeBuilder builder = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON); + Polygon geo = (Polygon) builder.buildGeometry(); + Geometry geometry = indexer.prepareForIndexing(geo); + Polygon testPolygon; + if (geometry instanceof Polygon) { + testPolygon = (Polygon) geometry; + } else if (geometry instanceof MultiPolygon) { + testPolygon = ((MultiPolygon) geometry).get(0); + } else { + throw new IllegalStateException("not a polygon"); + } + builder = LegacyGeoShapeQueryProcessor.geometryToShapeBuilder(testPolygon); + org.locationtech.spatial4j.shape.Rectangle box = builder.buildS4J().getBoundingBox(); + int minXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMinX()); + int minYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMinY()); + int maxXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMaxX()); + int maxYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMaxY()); + + double[] x = testPolygon.getPolygon().getLons(); + double[] y = testPolygon.getPolygon().getLats(); + + EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE, true); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); + Extent actualExtent = reader.getExtent(); + assertThat(actualExtent.minX(), equalTo(minXBox)); + assertThat(actualExtent.maxX(), equalTo(maxXBox)); + assertThat(actualExtent.minY(), equalTo(minYBox)); + assertThat(actualExtent.maxY(), equalTo(maxYBox)); + // polygon fully contained within box + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox)); + // relate + if (maxYBox - 1 >= minYBox) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1)); + } + if (maxXBox -1 >= minXBox) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox)); + } + // does not cross + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10)); + } + } + + public void testPacManPolygon() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // test cell crossing poly + ShapeTreeReader reader = geometryTreeReader(new Polygon(new LinearRing(py, px), Collections.emptyList()), + TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(-5, -6, 2, -2)); + } + + // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon + public void testPolygonWithHole() throws Exception { + Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), + Collections.singletonList(new LinearRing(new double[]{-10, 10, 10, -10, -10}, new double[]{-10, -10, 10, 10, -10}))); + + ShapeTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); + + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(6, -6, 6, -6)); // in the hole + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(25, -25, 25, -25)); // on the mainland + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(51, 51, 52, 52)); // outside of mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-60, -60, 60, 60)); // enclosing us completely + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(49, 49, 51, 51)); // overlapping the mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(9, 9, 11, 11)); // overlapping the hole + } + + public void testCombPolygon() throws Exception { + double[] px = {0, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 0, 0}; + double[] py = {0, 0, 20, 20, 0, 0, 20, 20, 0, 0, 30, 30, 0}; + + double[] hx = {21, 21, 29, 29, 21}; + double[] hy = {1, 20, 20, 1, 1}; + + Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); + ShapeTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); + // test cell crossing poly + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(5, 10, 5, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(15, 10, 15, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(25, 10, 25, 10)); + } + + public void testPacManClosedLineString() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // test cell crossing poly + ShapeTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); + } + + public void testPacManLineString() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; + + // test cell crossing poly + ShapeTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); + } + + public void testPacManPoints() throws Exception { + // pacman + List points = Arrays.asList( + new Point(0, 0), + new Point(5, 10), + new Point(9, 10), + new Point(10, 0), + new Point(9, -8), + new Point(0, -10), + new Point(-9, -8), + new Point(-10, 0), + new Point(-9, 10), + new Point(-5, 10) + ); + + + // candidate intersects cell + int xMin = 0; + int xMax = 11; + int yMin = -10; + int yMax = 9; + + // test cell crossing poly + ShapeTreeReader reader = geometryTreeReader(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(xMin, yMin, xMax, yMax)); + } + + public void testRandomMultiLineIntersections() throws IOException { + double extentSize = randomDoubleBetween(0.01, 10, true); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + MultiLine geometry = GeometryTestUtils.randomMultiLine(false); + geometry = (MultiLine) indexer.prepareForIndexing(geometry); + + ShapeTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + Extent readerExtent = reader.getExtent(); + + for (Line line : geometry) { + // extent that intersects edges + assertRelation(GeoRelation.QUERY_CROSSES, reader, bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize)); + + // extent that fully encloses a line in the MultiLine + Extent lineExtent = geometryTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); + assertRelation(GeoRelation.QUERY_CROSSES, reader, lineExtent); + + if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE + && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, + lineExtent.maxX() + 1, lineExtent.maxY() + 1)); + } + } + + // extent that fully encloses the MultiLine + assertRelation(GeoRelation.QUERY_CROSSES, reader, reader.getExtent()); + if (readerExtent.minX() != Integer.MIN_VALUE && readerExtent.maxX() != Integer.MAX_VALUE + && readerExtent.minY() != Integer.MIN_VALUE && readerExtent.maxY() != Integer.MAX_VALUE) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, + readerExtent.maxX() + 1, readerExtent.maxY() + 1)); + } + + } + + public void testRandomGeometryIntersection() throws IOException { + int testPointCount = randomIntBetween(100, 200); + Point[] testPoints = new Point[testPointCount]; + double extentSize = randomDoubleBetween(1, 10, true); + boolean[] intersects = new boolean[testPointCount]; + for (int i = 0; i < testPoints.length; i++) { + testPoints[i] = randomPoint(false); + } + + Geometry geometry = randomGeometryTreeGeometry(); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry preparedGeometry = indexer.prepareForIndexing(geometry); + + for (int i = 0; i < testPointCount; i++) { + int cur = i; + intersects[cur] = fold(preparedGeometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); + } + + for (int i = 0; i < testPointCount; i++) { + assertEquals(intersects[i], intersects(preparedGeometry, testPoints[i], extentSize)); + } + } + + private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) { + int xMin = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.max(x - extentSize, -180.0)); + int xMax = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.min(x + extentSize, 180.0)); + int yMin = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.max(y - extentSize, -90)); + int yMax = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.min(y + extentSize, 90)); + return Extent.fromPoints(xMin, yMin, xMax, yMax); + } + + private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { + GeoRelation relation = geometryTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) + .relate(bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize)); + return relation == GeoRelation.QUERY_CROSSES || relation == GeoRelation.QUERY_INSIDE; + } + + private static Geometry randomGeometryTreeGeometry() { + return randomGeometryTreeGeometry(0); + } + + private static Geometry randomGeometryTreeGeometry(int level) { + @SuppressWarnings("unchecked") Function geometry = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPoint, + level < 3 ? (b) -> randomGeometryTreeCollection(level + 1) : GeometryTestUtils::randomPoint // don't build too deep + ); + return geometry.apply(false); + } + + private static Geometry randomGeometryTreeCollection(int level) { + int size = ESTestCase.randomIntBetween(1, 10); + List shapes = new ArrayList<>(); + for (int i = 0; i < size; i++) { + shapes.add(randomGeometryTreeGeometry(level)); + } + return new GeometryCollection<>(shapes); + } + + protected abstract ShapeTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException; +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java b/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java new file mode 100644 index 0000000000000..17b7ce27d255e --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java @@ -0,0 +1,207 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.geo.GeoTestUtil; +import org.apache.lucene.geo.Tessellator; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.test.ESTestCase; +import org.locationtech.spatial4j.io.GeohashUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + + +// This test class is just for comparing the size of different encodings. +@LuceneTestCase.AwaitsFix(bugUrl = "this is just for reference") +public class EncodingComparisonTests extends ESTestCase { + + public void testRandomRectangle() throws Exception { + for (int i =0; i < 100; i++) { + org.apache.lucene.geo.Rectangle random = GeoTestUtil.nextBox(); + compareWriters(new Rectangle(random.minLon, random.maxLon, random.maxLat, random.minLat), + new GeoShapeCoordinateEncoder()); + } + } + + public void testRandomPolygon() throws Exception { + for (int i =0; i < 100; i++) { + org.apache.lucene.geo.Polygon random = GeoTestUtil.nextPolygon(); + while (true) { + try { + Tessellator.tessellate(random); + break; + } catch (Exception e) { + random = GeoTestUtil.nextPolygon(); + } + } + compareWriters(new Polygon(new LinearRing(random.getPolyLons(), random.getPolyLats()), + Collections.emptyList()), new GeoShapeCoordinateEncoder()); + } + } + + public void testRandomMultiPolygon() throws Exception { + for (int i =0; i < 100; i++) { + int n = TestUtil.nextInt(random(), 2, 10); + List polygons = new ArrayList<>(n); + for (int j =0; j < n; j++) { + org.apache.lucene.geo.Polygon random = GeoTestUtil.nextPolygon(); + while (true) { + try { + Tessellator.tessellate(random); + break; + } catch (Exception e) { + random = GeoTestUtil.nextPolygon(); + } + } + polygons.add(new Polygon(new LinearRing(random.getPolyLons(), random.getPolyLats()), + Collections.emptyList())); + } + MultiPolygon multiPolygon = new MultiPolygon(polygons); + compareWriters(multiPolygon, new GeoShapeCoordinateEncoder()); + } + } + + public void testRandomLine() throws Exception { + for (int i =0; i < 100; i++) { + double[] px = new double[2]; + double[] py = new double[2]; + for (int j =0; j < 2; j++) { + px[j] = GeoTestUtil.nextLongitude(); + px[j] = GeoTestUtil.nextLatitude(); + } + compareWriters(new Line(px, py), new GeoShapeCoordinateEncoder()); + } + } + + public void testRandomLines() throws Exception { + for (int i =0; i < 100; i++) { + int numPoints = TestUtil.nextInt(random(), 2, 1000); + double[] px = new double[numPoints]; + double[] py = new double[numPoints]; + for (int j =0; j < numPoints; j++) { + px[j] = GeoTestUtil.nextLongitude(); + px[j] = GeoTestUtil.nextLatitude(); + } + compareWriters(new Line(px, py), new GeoShapeCoordinateEncoder()); + } + } + + public void testRandomPoint() throws Exception { + for (int i =0; i < 100; i++) { + compareWriters(new Point(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude()), + new GeoShapeCoordinateEncoder()); + } + } + + public void testRandomPoints() throws Exception { + for (int i =0; i < 100; i++) { + int numPoints = TestUtil.nextInt(random(), 2, 1000); + List points = new ArrayList<>(numPoints); + + for (int j =0; j < numPoints; j++) { + points.add(new Point(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude())); + } + compareWriters(new MultiPoint(points), new GeoShapeCoordinateEncoder()); + } + } + + private void compareWriters(Geometry geometry, CoordinateEncoder encoder) throws Exception { + TriangleTreeWriter writer1 = new TriangleTreeWriter(geometry, encoder); + GeometryTreeWriter writer2 = new GeometryTreeWriter(geometry, encoder); + BytesStreamOutput output1 = new BytesStreamOutput(); + BytesStreamOutput output2 = new BytesStreamOutput(); + writer1.writeTo(output1); + writer2.writeTo(output2); + output1.close(); + output2.close(); + //String s1 = "Triangles: " + output1.bytes().length(); + //String s2 = "Edge tree: " + output2.bytes().length(); + // System.out.println(s1 + " / " + s2 + " / diff: " + (double) output1.bytes().length() / output2.bytes().length()); + + int maxPrecision = 5; + TriangleTreeReader reader1 = new TriangleTreeReader(output1.bytes().toBytesRef(), encoder); + //long t1 = System.nanoTime(); + int h1 = getHashesAtLevel(reader1, encoder, "", maxPrecision); + //long t2 = System.nanoTime(); + GeometryTreeReader reader2 = new GeometryTreeReader(output2.bytes().toBytesRef(), encoder); + //long t3 = System.nanoTime(); + int h2= getHashesAtLevel(reader2, encoder, "", maxPrecision); + //long t4 = System.nanoTime(); + assertEquals(h1, h2); + + //String s3 = "Triangles: " + h1; //(t2 - t1); + //String s4 = "Edge tree: " + h2; //(t4 - t3); + //System.out.println("Query: " + s3 + " / " + s4 + " / diff: " + (double) (t2 - t1) / (t4 - t3)); + } + + + private int getHashesAtLevel(ShapeTreeReader reader, CoordinateEncoder encoder, String hash, int maxPrecision) throws IOException { + int hits = 0; + String[] hashes = GeohashUtils.getSubGeohashes(hash); + for (int i =0; i < hashes.length; i++) { + Rectangle r = Geohash.toBoundingBox(hashes[i]); + Extent extent = Extent.fromPoints(encoder.encodeX(r.getMinLon()), + encoder.encodeY(r.getMinLat()), + encoder.encodeX(r.getMaxLon()), + encoder.encodeY(r.getMaxLat())); + GeoRelation rel = reader.relate(extent); + if (rel == GeoRelation.QUERY_CROSSES) { + if (hashes[i].length() == maxPrecision) { + hits++; + } else { + hits += getHashesAtLevel(reader, encoder, hashes[i], maxPrecision); + } + } else if (rel == GeoRelation.QUERY_INSIDE) { + if (hashes[i].length() == maxPrecision) { + hits++; + } else { + hits += getHashesAtLevel(hashes[i], maxPrecision); + } + } + } + return hits; + } + + private int getHashesAtLevel(String hash, int maxPrecision) { + int hits = 0; + String[] hashes = GeohashUtils.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + if (hashes[i].length() == maxPrecision) { + hits++; + } else { + hits += getHashesAtLevel(hashes[i], maxPrecision); + } + } + return hits; + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index c9d18f98f2ca0..94b33f122f8b5 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -18,345 +18,15 @@ */ package org.elasticsearch.common.geo; -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.GeometryCollection; -import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; -import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.index.mapper.GeoShapeIndexer; -import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.geo.RandomShapeGenerator; import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.Function; -import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; -import static org.elasticsearch.geo.GeometryTestUtils.fold; -import static org.elasticsearch.geo.GeometryTestUtils.randomPoint; -import static org.hamcrest.Matchers.equalTo; -public class GeometryTreeTests extends ESTestCase { +public class GeometryTreeTests extends AbstractTreeTestCase { - public void testRectangleShape() throws IOException { - for (int i = 0; i < 1000; i++) { - int minX = randomIntBetween(-80, 70); - int maxX = randomIntBetween(minX + 10, 80); - int minY = randomIntBetween(-80, 70); - int maxY = randomIntBetween(minY + 10, 80); - double[] x = new double[]{minX, maxX, maxX, minX, minX}; - double[] y = new double[]{minY, minY, maxY, maxY, minY}; - Geometry rectangle = randomBoolean() ? - new Polygon(new LinearRing(x, y), Collections.emptyList()) : new Rectangle(minX, maxX, maxY, minY); - GeometryTreeReader reader = geometryTreeReader(rectangle, TestCoordinateEncoder.INSTANCE); - - assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); - // encoder loses precision when casting to integer, so centroid is calculated using integer division here - assertThat(reader.getCentroidX(), equalTo((double) ((minX + maxX) / 2))); - assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY) / 2))); - - // box-query touches bottom-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), - minY - randomIntBetween(1, 180), minX, minY)); - // box-query touches bottom-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), - maxX + randomIntBetween(1, 180), minY)); - // box-query touches top-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), - maxY + randomIntBetween(1, 180))); - // box-query touches top-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, - maxY + randomIntBetween(1, 180))); - // box-query fully-enclosed inside rectangle - assertRelation(GeoRelation.QUERY_INSIDE,reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); - // box-query fully-contains poly - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), - minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); - // box-query half-in-half-out-right - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); - // box-query half-in-half-out-left - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, - (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); - // box-query half-in-half-out-top - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - maxX + randomIntBetween(1, 1000), maxY + randomIntBetween(1, 1000))); - // box-query half-in-half-out-bottom - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); - - // box-query outside to the right - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, - maxX + randomIntBetween(1001, 2000), maxY)); - // box-query outside to the left - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, - minX - randomIntBetween(1, 1000), maxY)); - // box-query outside to the top - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, - maxY + randomIntBetween(1001, 2000))); - // box-query outside to the bottom - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, - minY - randomIntBetween(1, 1000))); - } - } - - public void testSimplePolygon() throws IOException { - for (int iter = 0; iter < 1000; iter++) { - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "name"); - ShapeBuilder builder = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON); - Polygon geo = (Polygon) builder.buildGeometry(); - Geometry geometry = indexer.prepareForIndexing(geo); - Polygon testPolygon; - if (geometry instanceof Polygon) { - testPolygon = (Polygon) geometry; - } else if (geometry instanceof MultiPolygon) { - testPolygon = ((MultiPolygon) geometry).get(0); - } else { - throw new IllegalStateException("not a polygon"); - } - builder = LegacyGeoShapeQueryProcessor.geometryToShapeBuilder(testPolygon); - org.locationtech.spatial4j.shape.Rectangle box = builder.buildS4J().getBoundingBox(); - int minXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMinX()); - int minYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMinY()); - int maxXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMaxX()); - int maxYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMaxY()); - - double[] x = testPolygon.getPolygon().getLons(); - double[] y = testPolygon.getPolygon().getLats(); - - EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE, true); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); - Extent actualExtent = reader.getExtent(); - assertThat(actualExtent.minX(), equalTo(minXBox)); - assertThat(actualExtent.maxX(), equalTo(maxXBox)); - assertThat(actualExtent.minY(), equalTo(minYBox)); - assertThat(actualExtent.maxY(), equalTo(maxYBox)); - // polygon fully contained within box - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox)); - // relate - if (maxYBox - 1 >= minYBox) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1)); - } - if (maxXBox -1 >= minXBox) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox)); - } - // does not cross - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10)); - } - } - - public void testPacManPolygon() throws Exception { - // pacman - double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - - // test cell crossing poly - GeometryTreeReader reader = geometryTreeReader(new Polygon(new LinearRing(py, px), Collections.emptyList()), - TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(-5, -6, 2, -2)); - } - - // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon - public void testPolygonWithHole() throws Exception { - Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), - Collections.singletonList(new LinearRing(new double[]{-10, 10, 10, -10, -10}, new double[]{-10, -10, 10, 10, -10}))); - - GeometryTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); - - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(6, -6, 6, -6)); // in the hole - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(25, -25, 25, -25)); // on the mainland - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(51, 51, 52, 52)); // outside of mainland - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-60, -60, 60, 60)); // enclosing us completely - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(49, 49, 51, 51)); // overlapping the mainland - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(9, 9, 11, 11)); // overlapping the hole - } - - public void testCombPolygon() throws Exception { - double[] px = {0, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 0, 0}; - double[] py = {0, 0, 20, 20, 0, 0, 20, 20, 0, 0, 30, 30, 0}; - - double[] hx = {21, 21, 29, 29, 21}; - double[] hy = {1, 20, 20, 1, 1}; - - Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); - GeometryTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); - // test cell crossing poly - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(5, 10, 5, 10)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(15, 10, 15, 10)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(25, 10, 25, 10)); - } - - public void testPacManClosedLineString() throws Exception { - // pacman - double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - - // test cell crossing poly - GeometryTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); - } - - public void testPacManLineString() throws Exception { - // pacman - double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; - - // test cell crossing poly - GeometryTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); - } - - public void testPacManPoints() throws Exception { - // pacman - List points = Arrays.asList( - new Point(0, 0), - new Point(5, 10), - new Point(9, 10), - new Point(10, 0), - new Point(9, -8), - new Point(0, -10), - new Point(-9, -8), - new Point(-10, 0), - new Point(-9, 10), - new Point(-5, 10) - ); - - - // candidate intersects cell - int xMin = 0; - int xMax = 11; - int yMin = -10; - int yMax = 9; - - // test cell crossing poly - GeometryTreeReader reader = geometryTreeReader(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(xMin, yMin, xMax, yMax)); - } - - public void testRandomMultiLineIntersections() throws IOException { - double extentSize = randomDoubleBetween(0.01, 10, true); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - MultiLine geometry = GeometryTestUtils.randomMultiLine(false); - geometry = (MultiLine) indexer.prepareForIndexing(geometry); - - GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); - Extent readerExtent = reader.getExtent(); - - for (Line line : geometry) { - // extent that intersects edges - assertRelation(GeoRelation.QUERY_CROSSES, reader, bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize)); - - // extent that fully encloses a line in the MultiLine - Extent lineExtent = geometryTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); - assertRelation(GeoRelation.QUERY_CROSSES, reader, lineExtent); - - if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE - && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, - lineExtent.maxX() + 1, lineExtent.maxY() + 1)); - } - } - - // extent that fully encloses the MultiLine - assertRelation(GeoRelation.QUERY_CROSSES, reader, reader.getExtent()); - if (readerExtent.minX() != Integer.MIN_VALUE && readerExtent.maxX() != Integer.MAX_VALUE - && readerExtent.minY() != Integer.MIN_VALUE && readerExtent.maxY() != Integer.MAX_VALUE) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, - readerExtent.maxX() + 1, readerExtent.maxY() + 1)); - } - - } - - public void testRandomGeometryIntersection() throws IOException { - int testPointCount = randomIntBetween(100, 200); - Point[] testPoints = new Point[testPointCount]; - double extentSize = randomDoubleBetween(1, 10, true); - boolean[] intersects = new boolean[testPointCount]; - for (int i = 0; i < testPoints.length; i++) { - testPoints[i] = randomPoint(false); - } - - Geometry geometry = randomGeometryTreeGeometry(); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - Geometry preparedGeometry = indexer.prepareForIndexing(geometry); - - for (int i = 0; i < testPointCount; i++) { - int cur = i; - intersects[cur] = fold(preparedGeometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); - } - - for (int i = 0; i < testPointCount; i++) { - assertEquals(intersects[i], intersects(preparedGeometry, testPoints[i], extentSize)); - } - } - - private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) { - int xMin = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.max(x - extentSize, -180.0)); - int xMax = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.min(x + extentSize, 180.0)); - int yMin = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.max(y - extentSize, -90)); - int yMax = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.min(y + extentSize, 90)); - return Extent.fromPoints(xMin, yMin, xMax, yMax); - } - - private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { - GeoRelation relation = geometryTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) - .relate(bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize)); - return relation == GeoRelation.QUERY_CROSSES || relation == GeoRelation.QUERY_INSIDE; - } - - private static Geometry randomGeometryTreeGeometry() { - return randomGeometryTreeGeometry(0); - } - - private static Geometry randomGeometryTreeGeometry(int level) { - @SuppressWarnings("unchecked") Function geometry = ESTestCase.randomFrom( - GeometryTestUtils::randomLine, - GeometryTestUtils::randomPoint, - GeometryTestUtils::randomPolygon, - GeometryTestUtils::randomMultiLine, - GeometryTestUtils::randomMultiPoint, - level < 3 ? (b) -> randomGeometryTreeCollection(level + 1) : GeometryTestUtils::randomPoint // don't build too deep - ); - return geometry.apply(false); - } - - private static Geometry randomGeometryTreeCollection(int level) { - int size = ESTestCase.randomIntBetween(1, 10); - List shapes = new ArrayList<>(); - for (int i = 0; i < size; i++) { - shapes.add(randomGeometryTreeGeometry(level)); - } - return new GeometryCollection<>(shapes); - } - - private GeometryTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { + protected ShapeTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { GeometryTreeWriter writer = new GeometryTreeWriter(geometry, encoder); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java new file mode 100644 index 0000000000000..3f966e624a7a9 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -0,0 +1,36 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geometry.Geometry; + +import java.io.IOException; + + +public class TriangleTreeTests extends AbstractTreeTestCase { + + protected ShapeTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { + TriangleTreeWriter writer = new TriangleTreeWriter(geometry, encoder); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + output.close(); + return new TriangleTreeReader(output.bytes().toBytesRef(), encoder); + } +} From 3d5de4479913bab20f76f527eec8fe5852d05fb3 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 2 Dec 2019 18:51:19 +0100 Subject: [PATCH 38/62] [Geo] Do not create a tree reader for each document (#49720) Make tree readers reusable so we do not create a reader object for each document --- .../common/geo/GeometryTreeReader.java | 15 ++++++----- .../common/geo/TriangleTreeReader.java | 11 +++++--- .../index/fielddata/MultiGeoValues.java | 25 +++++++++++-------- .../plain/LatLonShapeDVAtomicFieldData.java | 5 +++- .../common/geo/EncodingComparisonTests.java | 7 +++--- .../common/geo/GeoTestUtils.java | 4 ++- .../common/geo/GeometryTreeTests.java | 4 ++- .../common/geo/TriangleTreeTests.java | 4 ++- .../bucket/geogrid/GeoGridTilerTests.java | 6 +++-- .../support/MissingValuesTests.java | 23 ++++++++++++++--- 10 files changed, 70 insertions(+), 34 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index b98c8a19fb5df..8a27e1fa897e3 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -27,7 +27,7 @@ import java.util.Optional; /** - * A tree reader. + * A reusable tree reader. * * This class supports checking bounding box * relations against the serialized geometry tree. @@ -35,13 +35,11 @@ public class GeometryTreeReader implements ShapeTreeReader { private static final int EXTENT_OFFSET = 8; - private final int startPosition; - private final ByteBufferStreamInput input; + private int startPosition; + private ByteBufferStreamInput input; private final CoordinateEncoder coordinateEncoder; - public GeometryTreeReader(BytesRef bytesRef, CoordinateEncoder coordinateEncoder) { - this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); - this.startPosition = 0; + public GeometryTreeReader(CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; } @@ -51,6 +49,11 @@ private GeometryTreeReader(ByteBufferStreamInput input, CoordinateEncoder coordi this.coordinateEncoder = coordinateEncoder; } + public void reset(BytesRef bytesRef) { + this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + this.startPosition = 0; + } + @Override public double getCentroidX() throws IOException { input.position(startPosition); diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index a39300f969d84..626234dac50a8 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -27,7 +27,7 @@ import static org.apache.lucene.geo.GeoUtils.orient; /** - * A tree reader for a previous serialized {@link org.elasticsearch.geometry.Geometry} using + * A tree reusable reader for a previous serialized {@link org.elasticsearch.geometry.Geometry} using * {@link TriangleTreeWriter}. * * This class supports checking bounding box @@ -36,16 +36,19 @@ public class TriangleTreeReader implements ShapeTreeReader { private final int extentOffset = 8; - private final ByteBufferStreamInput input; + private ByteBufferStreamInput input; private final CoordinateEncoder coordinateEncoder; private final Rectangle2D rectangle2D; - public TriangleTreeReader(BytesRef bytesRef, CoordinateEncoder coordinateEncoder) { - this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + public TriangleTreeReader(CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; this.rectangle2D = new Rectangle2D(); } + public void reset(BytesRef bytesRef) { + this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + } + /** * returns the bounding box of the geometry in the format [minX, maxX, minY, maxY]. */ diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index d18865ee656b5..78bc5bec75d19 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -26,6 +26,8 @@ import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.GeometryTreeReader; import org.elasticsearch.common.geo.GeometryTreeWriter; +import org.elasticsearch.common.geo.ShapeTreeReader; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.GeographyValidator; @@ -125,22 +127,19 @@ public String toString() { public static class GeoShapeValue implements GeoValue { private static final WellKnownText MISSING_GEOMETRY_PARSER = new WellKnownText(true, new GeographyValidator(true)); - private final GeometryTreeReader reader; - private final Extent extent; + private final ShapeTreeReader reader; - public GeoShapeValue(GeometryTreeReader reader) throws IOException { + public GeoShapeValue(ShapeTreeReader reader) { this.reader = reader; - this.extent = reader.getExtent(); - } - - public GeoShapeValue(Extent extent) { - this.reader = null; - this.extent = extent; } @Override public BoundingBox boundingBox() { - return new BoundingBox(extent, GeoShapeCoordinateEncoder.INSTANCE); + try { + return new BoundingBox(reader.getExtent(), GeoShapeCoordinateEncoder.INSTANCE); + } catch (IOException e) { + throw new IllegalStateException("unable to read bounding box", e); + } } /** @@ -185,7 +184,11 @@ public static GeoShapeValue missing(String missing) { try { Geometry geometry = MISSING_GEOMETRY_PARSER.fromWKT(missing); GeometryTreeWriter writer = new GeometryTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); - return new GeoShapeValue(writer.getExtent()); + BytesStreamOutput output = new BytesStreamOutput(); + writer.writeTo(output); + GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + reader.reset(output.bytes().toBytesRef()); + return new GeoShapeValue(reader); } catch (IOException | ParseException e) { throw new IllegalArgumentException("Can't apply missing value [" + missing + "]", e); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java index 7367f13d8bb72..80837f098caaa 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java @@ -62,6 +62,8 @@ public void close() { public MultiGeoValues getGeoValues() { try { final BinaryDocValues binaryValues = DocValues.getBinary(reader, fieldName); + final GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + final MultiGeoValues.GeoShapeValue geoShapeValue = new MultiGeoValues.GeoShapeValue(reader); return new MultiGeoValues() { @Override @@ -82,7 +84,8 @@ public ValuesSourceType valuesSourceType() { @Override public GeoValue nextValue() throws IOException { final BytesRef encoded = binaryValues.binaryValue(); - return new GeoShapeValue(new GeometryTreeReader(encoded, GeoShapeCoordinateEncoder.INSTANCE)); + reader.reset(encoded); + return geoShapeValue; } }; } catch (IOException e) { diff --git a/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java b/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java index 17b7ce27d255e..35df9c3bc3278 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java @@ -149,11 +149,13 @@ private void compareWriters(Geometry geometry, CoordinateEncoder encoder) throws // System.out.println(s1 + " / " + s2 + " / diff: " + (double) output1.bytes().length() / output2.bytes().length()); int maxPrecision = 5; - TriangleTreeReader reader1 = new TriangleTreeReader(output1.bytes().toBytesRef(), encoder); + TriangleTreeReader reader1 = new TriangleTreeReader(encoder); + reader1.reset(output1.bytes().toBytesRef()); //long t1 = System.nanoTime(); int h1 = getHashesAtLevel(reader1, encoder, "", maxPrecision); //long t2 = System.nanoTime(); - GeometryTreeReader reader2 = new GeometryTreeReader(output2.bytes().toBytesRef(), encoder); + GeometryTreeReader reader2 = new GeometryTreeReader(encoder); + reader2.reset(output1.bytes().toBytesRef()); //long t3 = System.nanoTime(); int h2= getHashesAtLevel(reader2, encoder, "", maxPrecision); //long t4 = System.nanoTime(); @@ -164,7 +166,6 @@ private void compareWriters(Geometry geometry, CoordinateEncoder encoder) throws //System.out.println("Query: " + s3 + " / " + s4 + " / diff: " + (double) (t2 - t1) / (t4 - t3)); } - private int getHashesAtLevel(ShapeTreeReader reader, CoordinateEncoder encoder, String hash, int maxPrecision) throws IOException { int hits = 0; String[] hashes = GeohashUtils.getSubGeohashes(hash); diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java index f294d075ba34b..c1754b3e8409c 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -44,7 +44,9 @@ public static GeometryTreeReader geometryTreeReader(Geometry geometry, Coordinat BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - return new GeometryTreeReader(output.bytes().toBytesRef(), encoder); + GeometryTreeReader reader = new GeometryTreeReader(encoder); + reader.reset(output.bytes().toBytesRef()); + return reader; } public static String toGeoJsonString(Geometry geometry) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java index 94b33f122f8b5..892571e1c4f3c 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java @@ -31,6 +31,8 @@ protected ShapeTreeReader geometryTreeReader(Geometry geometry, CoordinateEncode BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - return new GeometryTreeReader(output.bytes().toBytesRef(), encoder); + GeometryTreeReader reader = new GeometryTreeReader(encoder); + reader.reset(output.bytes().toBytesRef()); + return reader; } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java index 3f966e624a7a9..c0d9c76d18298 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -31,6 +31,8 @@ protected ShapeTreeReader geometryTreeReader(Geometry geometry, CoordinateEncode BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - return new TriangleTreeReader(output.bytes().toBytesRef(), encoder); + TriangleTreeReader reader = new TriangleTreeReader(encoder); + reader.reset(output.bytes().toBytesRef()); + return reader; } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java index 4a423a57b10c3..da96304fbb1bd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -56,7 +56,8 @@ public void testGeoTile() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), GeoShapeCoordinateEncoder.INSTANCE); + GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + reader.reset(output.bytes().toBytesRef()); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); long[] values = new long[16]; @@ -151,7 +152,8 @@ public void testGeoHash() throws Exception { BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(output.bytes().toBytesRef(), GeoShapeCoordinateEncoder.INSTANCE); + GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + reader.reset(output.bytes().toBytesRef()); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); long[] values = new long[1024]; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index 38ad0742c5dd2..f9f5c995f7d40 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -26,14 +26,19 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.TestUtil; -import org.elasticsearch.common.geo.Extent; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.GeometryTreeReader; +import org.elasticsearch.common.geo.TriangleTreeWriter; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.geo.RandomShapeGenerator; import java.io.IOException; import java.util.Arrays; @@ -401,10 +406,16 @@ public ValuesSourceType valuesSourceType() { public void testMissingGeoShapes() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final MultiGeoValues.GeoShapeValue[][] values = new MultiGeoValues.GeoShapeValue[numDocs][]; + GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); for (int i = 0; i < numDocs; ++i) { values[i] = new MultiGeoValues.GeoShapeValue[random().nextInt(4)]; for (int j = 0; j < values[i].length; ++j) { - values[i][j] = new MultiGeoValues.GeoShapeValue(Extent.fromPoint(randomInt(), randomInt())); + ShapeBuilder builder = RandomShapeGenerator.createShape(random()); + BytesStreamOutput outputStream = new BytesStreamOutput(); + TriangleTreeWriter writer = new TriangleTreeWriter(builder.buildGeometry(), GeoShapeCoordinateEncoder.INSTANCE); + writer.writeTo(outputStream); + reader.reset(outputStream.bytes().toBytesRef()); + values[i][j] = new MultiGeoValues.GeoShapeValue(reader); } } MultiGeoValues asGeoValues = new MultiGeoValues() { @@ -434,8 +445,12 @@ public ValuesSourceType valuesSourceType() { return CoreValuesSourceType.GEOSHAPE; } }; - final MultiGeoValues.GeoShapeValue missing = new MultiGeoValues.GeoShapeValue( - Extent.fromPoint(randomInt(), randomInt())); + ShapeBuilder builder = RandomShapeGenerator.createShape(random()); + BytesStreamOutput outputStream = new BytesStreamOutput(); + TriangleTreeWriter writer = new TriangleTreeWriter(builder.buildGeometry(), GeoShapeCoordinateEncoder.INSTANCE); + writer.writeTo(outputStream); + reader.reset(outputStream.bytes().toBytesRef()); + final MultiGeoValues.GeoShapeValue missing = new MultiGeoValues.GeoShapeValue(reader); MultiGeoValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); for (int i = 0; i < numDocs; ++i) { assertTrue(withMissingReplaced.advanceExact(i)); From 0a904773257d385061169991517b372a0589139e Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 3 Dec 2019 09:32:14 +0100 Subject: [PATCH 39/62] Improve resiliency and performance of geogrid aggregation over geoshapes (#49646) --- .../elasticsearch/geometry/utils/Geohash.java | 18 +- .../bucket/geogrid/CellIdSource.java | 109 +++++++-- .../bucket/geogrid/GeoGridTiler.java | 210 +++++++++++------- .../bucket/geogrid/GeoGridTilerTests.java | 116 +++++----- 4 files changed, 298 insertions(+), 155 deletions(-) diff --git a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java index f0924905aea5d..ffd55a45fe5f9 100644 --- a/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java +++ b/libs/geo/src/main/java/org/elasticsearch/geometry/utils/Geohash.java @@ -111,6 +111,16 @@ public static Rectangle toBoundingBox(final String geohash) { } } + /** Array of geohashes 1 level below the baseGeohash. Sorted. */ + public static String[] getSubGeohashes(String baseGeohash) { + String[] hashes = new String[BASE_32.length]; + for (int i = 0; i < BASE_32.length; i++) {//note: already sorted + char c = BASE_32[i]; + hashes[i] = baseGeohash+c; + } + return hashes; + } + /** * Calculate all neighbors of a given geohash cell. * @@ -215,6 +225,13 @@ public static final String getNeighbor(String geohash, int level, int dx, int dy } } + /** + * Encode a string geohash to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) + */ + public static final long longEncode(String hash) { + return longEncode(hash, hash.length()); + } + /** * Encode lon/lat to the geohash based long format (lon/lat interleaved, 4 least significant bits = level) */ @@ -311,7 +328,6 @@ private static long encodeLatLon(final double lat, final double lon) { return BitUtil.interleave(latEnc, lonEnc) >>> 2; } - /** encode latitude to integer */ public static int encodeLatitude(double latitude) { // the maximum possible value cannot be encoded without overflow diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java index 8c2335a9d556c..5ca51ac96c070 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java @@ -57,7 +57,21 @@ public boolean isFloatingPoint() { @Override public SortedNumericDocValues longValues(LeafReaderContext ctx) { - return new CellValues(valuesSource.geoValues(ctx), precision, encoder); + MultiGeoValues geoValues = valuesSource.geoValues(ctx); + if (precision == 0) { + // special case, precision 0 is the whole world + return new AllCellValues(geoValues, encoder); + } + ValuesSourceType vs = geoValues.valuesSourceType(); + if (CoreValuesSourceType.GEOPOINT == vs) { + // docValues are geo points + return new GeoPointCellValues(geoValues, precision, encoder); + } else if (CoreValuesSourceType.GEOSHAPE == vs || CoreValuesSourceType.GEO == vs) { + // docValues are geo shapes + return new GeoShapeCellValues(geoValues, precision, encoder); + } else { + throw new IllegalArgumentException("unsupported geo type"); + } } @Override @@ -70,40 +84,71 @@ public SortedBinaryDocValues bytesValues(LeafReaderContext ctx) { throw new UnsupportedOperationException(); } - private static class CellValues extends AbstractSortingNumericDocValues { + /** Sorted numeric doc values for geo shapes */ + protected static class GeoShapeCellValues extends AbstractSortingNumericDocValues { private MultiGeoValues geoValues; private int precision; private GeoGridTiler tiler; - protected CellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { + protected GeoShapeCellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { this.geoValues = geoValues; this.precision = precision; this.tiler = tiler; } + protected void resizeCell(int newSize) { + resize(newSize); + } + + protected void add(int idx, long value) { + values[idx] = value; + } + + // for testing + protected long[] getValues() { + return values; + } + @Override public boolean advanceExact(int docId) throws IOException { if (geoValues.advanceExact(docId)) { ValuesSourceType vs = geoValues.valuesSourceType(); - if (CoreValuesSourceType.GEOPOINT == vs) { - resize(geoValues.docValueCount()); - for (int i = 0; i < docValueCount(); ++i) { - MultiGeoValues.GeoValue target = geoValues.nextValue(); - values[i] = tiler.encode(target.lon(), target.lat(), precision); - } - } else if (CoreValuesSourceType.GEOSHAPE == vs || CoreValuesSourceType.GEO == vs) { + MultiGeoValues.GeoValue target = geoValues.nextValue(); + // TODO(talevy): determine reasonable circuit-breaker here + resize(0); + tiler.setValues(this, target, precision); + sort(); + return true; + } else { + return false; + } + } + } + + /** Sorted numeric doc values for geo points */ + protected static class GeoPointCellValues extends AbstractSortingNumericDocValues { + private MultiGeoValues geoValues; + private int precision; + private GeoGridTiler tiler; + + protected GeoPointCellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { + this.geoValues = geoValues; + this.precision = precision; + this.tiler = tiler; + } + + // for testing + protected long[] getValues() { + return values; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + if (geoValues.advanceExact(docId)) { + resize(geoValues.docValueCount()); + for (int i = 0; i < docValueCount(); ++i) { MultiGeoValues.GeoValue target = geoValues.nextValue(); - // TODO(talevy): determine reasonable circuit-breaker here - // must resize array to contain the upper-bound of matching cells, which - // is the number of tiles that overlap the shape's bounding-box. No need - // to be concerned with original docValueCount since shape doc-values are - // single-valued. - resize((int) tiler.getBoundingTileCount(target, precision)); - int matched = tiler.setValues(values, target, precision); - // must truncate array to only contain cells that actually intersected shape - resize(matched); - } else { - throw new IllegalArgumentException("unsupported geo type"); + values[i] = tiler.encode(target.lon(), target.lat(), precision); } sort(); return true; @@ -112,4 +157,26 @@ public boolean advanceExact(int docId) throws IOException { } } } + + /** Sorted numeric doc values for precision 0 */ + protected static class AllCellValues extends AbstractSortingNumericDocValues { + private MultiGeoValues geoValues; + + protected AllCellValues(MultiGeoValues geoValues, GeoGridTiler tiler) { + this.geoValues = geoValues; + resize(1); + values[0] = tiler.encode(0, 0, 0); + } + + // for testing + protected long[] getValues() { + return values; + } + + @Override + public boolean advanceExact(int docId) throws IOException { + resize(1); + return geoValues.advanceExact(docId); + } + } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java index 3741a87384cb7..eddb0ce33fddf 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -37,15 +37,6 @@ public interface GeoGridTiler { */ long encode(double x, double y, int precision); - /** - * computes the number of tiles for a specific precision that the geo value's - * bounding-box is contained within. - * - * @param geoValue the input shape - * @param precision the tile zoom-level - */ - long getBoundingTileCount(MultiGeoValues.GeoValue geoValue, int precision); - /** * * @param docValues the array of long-encoded bucket keys to fill @@ -54,7 +45,7 @@ public interface GeoGridTiler { * * @return the number of tiles the geoValue intersects */ - int setValues(long[] docValues, MultiGeoValues.GeoValue geoValue, int precision); + int setValues(CellIdSource.GeoShapeCellValues docValues, MultiGeoValues.GeoValue geoValue, int precision); class GeoHashGridTiler implements GeoGridTiler { public static final GeoHashGridTiler INSTANCE = new GeoHashGridTiler(); @@ -67,37 +58,92 @@ public long encode(double x, double y, int precision) { } @Override - public long getBoundingTileCount(MultiGeoValues.GeoValue geoValue, int precision) { + public int setValues(CellIdSource.GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, int precision) { MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); - // find minimum (x,y) of geo-hash-cell that contains (bounds.minX, bounds.minY) - String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); - Rectangle geoHashCell = Geohash.toBoundingBox(hash); - long numLonCells = Math.max(1, (long) Math.ceil( - (bounds.maxX() - geoHashCell.getMinX()) / Geohash.lonWidthInDegrees(precision))); - long numLatCells = Math.max(1, (long) Math.ceil( - (bounds.maxY() - geoHashCell.getMinY()) / Geohash.latHeightInDegrees(precision))); - return numLonCells * numLatCells; + assert bounds.minX() <= bounds.maxX(); + long numLonCells = (long) ((bounds.maxX() - bounds.minX()) / Geohash.lonWidthInDegrees(precision)); + long numLatCells = (long) ((bounds.maxY() - bounds.minY()) / Geohash.latHeightInDegrees(precision)); + long count = (numLonCells + 1) * (numLatCells + 1); + if (count == 1) { + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + values.resizeCell(1); + values.add(0, Geohash.longEncode(hash)); + return 1; + } else if (count <= precision) { + return setValuesByBruteForceScan(values, geoValue, precision, bounds); + } else { + return setValuesByRasterization("", values, 0, precision, geoValue, bounds); + } } - @Override - public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precision) { - MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + protected int setValuesByBruteForceScan(CellIdSource.GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, + int precision, MultiGeoValues.BoundingBox bounds) { + // TODO: This way to discover cells inside of a bounding box seems not to work as expected. I can + // see that eventually we will be visiting twice the same cell which should not happen. int idx = 0; - // find minimum (x,y) of geo-hash-cell that contains (bounds.minX, bounds.minY) - String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); - Rectangle geoHashCell = Geohash.toBoundingBox(hash); - for (double i = geoHashCell.getMinX(); i < bounds.maxX(); i+= Geohash.lonWidthInDegrees(precision)) { - for (double j = geoHashCell.getMinY(); j < bounds.maxY(); j += Geohash.latHeightInDegrees(precision)) { + String min = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + String max = Geohash.stringEncode(bounds.maxX(), bounds.maxY(), precision); + double minY = Geohash.decodeLatitude(min); + double minX = Geohash.decodeLongitude(min); + double maxY = Geohash.decodeLatitude(max); + double maxX = Geohash.decodeLongitude(max); + for (double i = minX; i <= maxX; i += Geohash.lonWidthInDegrees(precision)) { + for (double j = minY; j <= maxY; j += Geohash.latHeightInDegrees(precision)) { Rectangle rectangle = Geohash.toBoundingBox(Geohash.stringEncode(i, j, precision)); GeoRelation relation = geoValue.relate(rectangle); if (relation != GeoRelation.QUERY_DISJOINT) { - values[idx++] = encode(i, j, precision); + values.resizeCell(idx + 1); + values.add(idx++, encode(i, j, precision)); } } } - return idx; } + + protected int setValuesByRasterization(String hash, CellIdSource.GeoShapeCellValues values, int valuesIndex, + int targetPrecision, MultiGeoValues.GeoValue geoValue, + MultiGeoValues.BoundingBox shapeBounds) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + Rectangle rectangle = Geohash.toBoundingBox(hashes[i]); + if (shapeBounds.minX() == rectangle.getMaxX() || + shapeBounds.maxY() == rectangle.getMinY()) { + continue; + } + GeoRelation relation = geoValue.relate(rectangle); + if (relation == GeoRelation.QUERY_CROSSES) { + if (hashes[i].length() == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + valuesIndex = + setValuesByRasterization(hashes[i], values, valuesIndex, targetPrecision, geoValue, shapeBounds); + } + } else if (relation == GeoRelation.QUERY_INSIDE) { + if (hashes[i].length() == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + values.resizeCell(valuesIndex + (int) Math.pow(32, targetPrecision - hash.length()) + 1); + valuesIndex = setValuesForFullyContainedTile(hashes[i],values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } + + private int setValuesForFullyContainedTile(String hash, CellIdSource.GeoShapeCellValues values, + int valuesIndex, int targetPrecision) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + if (hashes[i].length() == targetPrecision) { + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + valuesIndex = setValuesForFullyContainedTile(hashes[i], values, valuesIndex, targetPrecision); + } + } + return valuesIndex; + } } class GeoTileGridTiler implements GeoGridTiler { @@ -111,19 +157,24 @@ public long encode(double x, double y, int precision) { } @Override - public long getBoundingTileCount(MultiGeoValues.GeoValue geoValue, int precision) { + public int setValues(CellIdSource.GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, int precision) { MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + assert bounds.minX() <= bounds.maxX(); final double tiles = 1 << precision; int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); - return (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1); - } - - @Override - public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precision) { - return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue); + int count = (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1); + if (count == 1) { + values.resizeCell(1); + values.add(0, GeoTileUtils.longEncodeTiles(precision, minXTile, minYTile)); + return 1; + } else if (count <= precision) { + return setValuesByBruteForceScan(values, geoValue, precision, minXTile, minYTile, maxXTile, maxYTile); + } else { + return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue, bounds); + } } /** @@ -133,73 +184,72 @@ public int setValues(long[] values, MultiGeoValues.GeoValue geoValue, int precis * @param precision the target precision to split the shape up into * @return the number of buckets the geoValue is found in */ - public int setValuesByBruteForceScan(long[] values, MultiGeoValues.GeoValue geoValue, int precision) { - MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); - - final double tiles = 1 << precision; - int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); - int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); - int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); - int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + protected int setValuesByBruteForceScan(CellIdSource.GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, + int precision, int minXTile, int minYTile, int maxXTile, int maxYTile) { int idx = 0; for (int i = minXTile; i <= maxXTile; i++) { for (int j = minYTile; j <= maxYTile; j++) { Rectangle rectangle = GeoTileUtils.toBoundingBox(i, j, precision); if (geoValue.relate(rectangle) != GeoRelation.QUERY_DISJOINT) { - values[idx++] = GeoTileUtils.longEncodeTiles(precision, i, j); + values.resizeCell(idx + 1); + values.add(idx++, GeoTileUtils.longEncodeTiles(precision, i, j)); } } } - return idx; } - private int setValuesByRasterization(int xTile, int yTile, int zTile, long[] values, int valuesIndex, int targetPrecision, - MultiGeoValues.GeoValue geoValue) { - Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, zTile); - MultiGeoValues.BoundingBox shapeBounds = geoValue.boundingBox(); - if (shapeBounds.minX() == rectangle.getMaxX() || - shapeBounds.maxY() == rectangle.getMinY()) { - return valuesIndex; - } - GeoRelation relation = geoValue.relate(rectangle); - if (zTile == targetPrecision) { - if (GeoRelation.QUERY_DISJOINT != relation) { - values[valuesIndex++] = GeoTileUtils.longEncodeTiles(zTile, xTile, yTile); - } - return valuesIndex; - } - - if (GeoRelation.QUERY_INSIDE == relation) { - return setValuesForFullyContainedTile(xTile, yTile, zTile, values, valuesIndex, targetPrecision); - } - if (GeoRelation.QUERY_CROSSES == relation) { - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int nextX = 2 * xTile + i; - int nextY = 2 * yTile + j; - valuesIndex = setValuesByRasterization(nextX, nextY, zTile + 1, values, valuesIndex, targetPrecision, geoValue); + protected int setValuesByRasterization(int xTile, int yTile, int zTile, CellIdSource.GeoShapeCellValues values, + int valuesIndex, int targetPrecision, MultiGeoValues.GeoValue geoValue, + MultiGeoValues.BoundingBox shapeBounds) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + Rectangle rectangle = GeoTileUtils.toBoundingBox(nextX, nextY, zTile); + // TODO: this looks hacky, maybe the relate method should handle it? + if (shapeBounds.minX() == rectangle.getMaxX() || + shapeBounds.maxY() == rectangle.getMinY()) { + continue; + } + GeoRelation relation = geoValue.relate(rectangle); + if (GeoRelation.QUERY_INSIDE == relation) { + if (zTile == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + values.resizeCell(valuesIndex + (int) Math.pow(4, targetPrecision - zTile) + 1); + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } else if (GeoRelation.QUERY_CROSSES == relation) { + if (zTile == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, + targetPrecision, geoValue, shapeBounds); + } } } } - return valuesIndex; } - private int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, long[] values, int valuesIndex, int targetPrecision) { - if (zTile == targetPrecision) { - values[valuesIndex] = GeoTileUtils.longEncodeTiles(zTile, xTile, yTile); - return valuesIndex + 1; - } - + private int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, + CellIdSource.GeoShapeCellValues values, int valuesIndex, int targetPrecision) { + zTile++; for (int i = 0; i < 2; i++) { for (int j = 0; j < 2; j++) { int nextX = 2 * xTile + i; int nextY = 2 * yTile + j; - valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile + 1, values, valuesIndex, targetPrecision); + if (zTile == targetPrecision) { + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } } } - return valuesIndex; } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java index da96304fbb1bd..5463a3af08d74 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -25,9 +25,7 @@ import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPolygon; import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.mapper.GeoShapeIndexer; @@ -60,82 +58,96 @@ public void testGeoTile() throws Exception { reader.reset(output.bytes().toBytesRef()); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); - long[] values = new long[16]; - // test shape within tile bounds { + CellIdSource.GeoShapeCellValues values = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); int count = GEOTILE.setValues(values, value, 13); - assertThat(GEOTILE.getBoundingTileCount(value, 13), equalTo(1L)); assertThat(count, equalTo(1)); } { + CellIdSource.GeoShapeCellValues values = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); int count = GEOTILE.setValues(values, value, 14); - assertThat(GEOTILE.getBoundingTileCount(value, 14), equalTo(4L)); assertThat(count, equalTo(4)); } { + CellIdSource.GeoShapeCellValues values = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); int count = GEOTILE.setValues(values, value, 15); - assertThat(GEOTILE.getBoundingTileCount(value, 15), equalTo(16L)); assertThat(count, equalTo(16)); } } public void testGeoTileSetValuesBruteAndRecursiveMultiline() throws Exception { - int precision = randomIntBetween(0, 10); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); MultiLine geometry = GeometryTestUtils.randomMultiLine(false); - geometry = (MultiLine) indexer.prepareForIndexing(geometry); - GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); - MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); - int upperBound = (int) GEOTILE.getBoundingTileCount(value, precision); - long[] recursiveValues = new long[upperBound]; - long[] bruteForceValues = new long[upperBound]; - int recursiveCount = GEOTILE.setValues(recursiveValues, value, precision); - int bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision); - Arrays.sort(recursiveValues); - Arrays.sort(bruteForceValues); - assertThat(recursiveCount, equalTo(bruteForceCount)); - assertArrayEquals(recursiveValues, bruteForceValues); + checkGeoTileSetValuesBruteAndRecursive(geometry); + // checkGeoHashSetValuesBruteAndRecursive(geometry); } public void testGeoTileSetValuesBruteAndRecursivePolygon() throws Exception { - int precision = randomIntBetween(0, 10); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); Geometry geometry = GeometryTestUtils.randomPolygon(false); + checkGeoTileSetValuesBruteAndRecursive(geometry); + // checkGeoHashSetValuesBruteAndRecursive(geometry); + } + + public void testGeoTileSetValuesBruteAndRecursivePoints() throws Exception { + Geometry geometry = randomBoolean() ? GeometryTestUtils.randomPoint(false) : GeometryTestUtils.randomMultiPoint(false); + checkGeoTileSetValuesBruteAndRecursive(geometry); + // checkGeoHashSetValuesBruteAndRecursive(geometry); + } + + private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Exception { + int precision = randomIntBetween(1, 10); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); geometry = indexer.prepareForIndexing(geometry); - // TODO: support multipolygons. for now just extract first polygon - if (geometry.type() == ShapeType.MULTIPOLYGON) { - geometry = ((MultiPolygon) geometry).get(0); - } GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); - int upperBound = (int) GEOTILE.getBoundingTileCount(value, precision); - long[] recursiveValues = new long[upperBound]; - long[] bruteForceValues = new long[upperBound]; - int recursiveCount = GEOTILE.setValues(recursiveValues, value, precision); - int bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision); - Arrays.sort(recursiveValues); - Arrays.sort(bruteForceValues); - assertThat(recursiveCount, equalTo(bruteForceCount)); - assertArrayEquals(recursiveValues, bruteForceValues); + CellIdSource.GeoShapeCellValues recursiveValues = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); + int recursiveCount; + { + recursiveCount = GEOTILE.setValuesByRasterization(0, 0, 0, recursiveValues, 0, + precision, value, value.boundingBox()); + } + CellIdSource.GeoShapeCellValues bruteForceValues = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); + int bruteForceCount; + { + final double tiles = 1 << precision; + MultiGeoValues.BoundingBox bounds = value.boundingBox(); + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision, minXTile, minYTile, maxXTile, maxYTile); + } + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); + long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); + Arrays.sort(recursive); + Arrays.sort(bruteForce); + assertArrayEquals(geometry.toString(), recursive, bruteForce); } - public void testGeoTileSetValuesBruteAndRecursivePoints() throws Exception { - int precision = randomIntBetween(0, 10); + private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Exception { + int precision = randomIntBetween(1, 4); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - Geometry geometry = randomBoolean() ? GeometryTestUtils.randomPoint(false) : GeometryTestUtils.randomMultiPoint(false); geometry = indexer.prepareForIndexing(geometry); GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); - int upperBound = (int) GEOTILE.getBoundingTileCount(value, precision); - long[] recursiveValues = new long[upperBound]; - long[] bruteForceValues = new long[upperBound]; - int recursiveCount = GEOTILE.setValues(recursiveValues, value, precision); - int bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision); - Arrays.sort(recursiveValues); - Arrays.sort(bruteForceValues); - assertThat(recursiveCount, equalTo(bruteForceCount)); - assertArrayEquals(recursiveValues, bruteForceValues); + CellIdSource.GeoShapeCellValues recursiveValues = new CellIdSource.GeoShapeCellValues(null, precision, GEOHASH); + int recursiveCount; + { + recursiveCount = GEOHASH.setValuesByRasterization("", recursiveValues, 0, precision, value, value.boundingBox()); + } + CellIdSource.GeoShapeCellValues bruteForceValues = new CellIdSource.GeoShapeCellValues(null, precision, GEOHASH); + int bruteForceCount; + { + MultiGeoValues.BoundingBox bounds = value.boundingBox(); + bruteForceCount = GEOHASH.setValuesByBruteForceScan(bruteForceValues, value, precision, bounds); + } + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); + long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); + Arrays.sort(recursive); + Arrays.sort(bruteForce); + assertArrayEquals(geometry.toString(), recursive, bruteForce); } public void testGeoHash() throws Exception { @@ -156,22 +168,20 @@ public void testGeoHash() throws Exception { reader.reset(output.bytes().toBytesRef()); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); - long[] values = new long[1024]; - // test shape within tile bounds { + CellIdSource.GeoShapeCellValues values = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); int count = GEOHASH.setValues(values, value, 5); - assertThat(GEOHASH.getBoundingTileCount(value, 5), equalTo(1L)); assertThat(count, equalTo(1)); } { + CellIdSource.GeoShapeCellValues values = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); int count = GEOHASH.setValues(values, value, 6); - assertThat(GEOHASH.getBoundingTileCount(value, 6), equalTo(32L)); assertThat(count, equalTo(32)); } { + CellIdSource.GeoShapeCellValues values = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); int count = GEOHASH.setValues(values, value, 7); - assertThat(GEOHASH.getBoundingTileCount(value, 7), equalTo(1024L)); assertThat(count, equalTo(1024)); } } From eca332899c04d86d14ddb3fba3e6aaeed972e199 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 3 Dec 2019 15:03:13 -0800 Subject: [PATCH 40/62] fixes assertion of shape-count in GeometryTreeReader (#49810) * fixes assertion of shape-count in GeometryTreeReader There is an assertion of the number of shapes that exist in a geometry with no prepended extent. This is because it is owned by the sub-tree to read the extent. This assertion is run at test-time, but not at run-time. Somehow runtime tests never caught this issue --- .../java/org/elasticsearch/common/geo/GeometryTreeReader.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 8a27e1fa897e3..db4637de88e79 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -73,7 +73,8 @@ public Extent getExtent() throws IOException { if (extent != null) { return extent; } - assert input.readVInt() == 1; + int numShapes = input.readVInt(); + assert numShapes == 1; ShapeType shapeType = input.readEnum(ShapeType.class); ShapeTreeReader reader = getReader(shapeType, coordinateEncoder, input); return reader.getExtent(); From 87344b117d83e2cc16735d82a50f1d001ce92468 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 4 Dec 2019 07:23:35 -0800 Subject: [PATCH 41/62] use ShapeType ordinals when deserializing shape trees by type (#49811) StreamInput#readEnum allocates objects when retrieving class constants from the ordinal value it serialized in writeEnum. This change by-passes this call to reduce the amount of objects allocated at read-time. --- .../common/geo/GeometryTreeReader.java | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index db4637de88e79..236ff568f1c43 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -75,8 +75,9 @@ public Extent getExtent() throws IOException { } int numShapes = input.readVInt(); assert numShapes == 1; - ShapeType shapeType = input.readEnum(ShapeType.class); - ShapeTreeReader reader = getReader(shapeType, coordinateEncoder, input); + // read ShapeType ordinal to avoid readEnum allocations + int shapeTypeOrdinal = input.readVInt(); + ShapeTreeReader reader = getReader(shapeTypeOrdinal, coordinateEncoder, input); return reader.getExtent(); } @@ -102,8 +103,9 @@ public GeoRelation relate(Extent extent) throws IOException { int pos = input.readVInt(); nextPosition = input.position() + pos; } - ShapeType shapeType = input.readEnum(ShapeType.class); - ShapeTreeReader reader = getReader(shapeType, coordinateEncoder, input); + // read ShapeType ordinal to avoid readEnum allocations + int shapeTypeOrdinal = input.readVInt(); + ShapeTreeReader reader = getReader(shapeTypeOrdinal, coordinateEncoder, input); GeoRelation shapeRelation = reader.relate(extent); if (GeoRelation.QUERY_CROSSES == shapeRelation || (GeoRelation.QUERY_DISJOINT == shapeRelation && GeoRelation.QUERY_INSIDE == relation) @@ -117,21 +119,17 @@ public GeoRelation relate(Extent extent) throws IOException { return relation; } - private static ShapeTreeReader getReader(ShapeType shapeType, CoordinateEncoder coordinateEncoder, ByteBufferStreamInput input) + private static ShapeTreeReader getReader(int shapeTypeOrdinal, CoordinateEncoder coordinateEncoder, ByteBufferStreamInput input) throws IOException { - switch (shapeType) { - case POLYGON: - return new PolygonTreeReader(input); - case POINT: - case MULTIPOINT: - return new Point2DReader(input); - case LINESTRING: - case MULTILINESTRING: - return new EdgeTreeReader(input, false); - case GEOMETRYCOLLECTION: - return new GeometryTreeReader(input, coordinateEncoder); - default: - throw new UnsupportedOperationException("unsupported shape type [" + shapeType + "]"); + if (shapeTypeOrdinal == ShapeType.POLYGON.ordinal()) { + return new PolygonTreeReader(input); + } else if (shapeTypeOrdinal == ShapeType.POINT.ordinal() || shapeTypeOrdinal == ShapeType.MULTIPOINT.ordinal()) { + return new Point2DReader(input); + } else if (shapeTypeOrdinal == ShapeType.LINESTRING.ordinal() || shapeTypeOrdinal == ShapeType.MULTILINESTRING.ordinal()) { + return new EdgeTreeReader(input, false); + } else if (shapeTypeOrdinal == ShapeType.GEOMETRYCOLLECTION.ordinal()) { + return new GeometryTreeReader(input, coordinateEncoder); } + throw new UnsupportedOperationException("unsupported shape type ordinal [" + shapeTypeOrdinal + "]"); } } From 85f1e7543ac0b9a1a69e3ed3c75e02e999ca2a02 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 4 Dec 2019 09:17:05 -0800 Subject: [PATCH 42/62] remove usage of Extent in ShapeTreeReader#relate logic (#49816) Extent is mostly existing for the bounding-box queries because it keeps track of more information for easy arithmetic. Since shape readers do not need to be concerned with date-line crossing, its usage is not required and by avoiding it, fewer allocations are made at runtime --- .../common/geo/EdgeTreeReader.java | 135 ++++++++---------- .../common/geo/GeometryTreeReader.java | 23 ++- .../common/geo/Point2DReader.java | 10 +- .../common/geo/PolygonTreeReader.java | 6 +- .../common/geo/ShapeTreeReader.java | 2 +- .../common/geo/TriangleTreeReader.java | 8 +- .../index/fielddata/MultiGeoValues.java | 3 +- .../common/geo/AbstractTreeTestCase.java | 4 +- .../common/geo/EncodingComparisonTests.java | 3 +- .../common/geo/GeoTestUtils.java | 2 +- 10 files changed, 93 insertions(+), 103 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java index a67b296398b31..b212f29cf8632 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import java.io.IOException; -import java.util.Optional; import static org.apache.lucene.geo.GeoUtils.lineCrossesLineWithBoundary; @@ -30,28 +29,19 @@ * serialized with the {@link EdgeTreeWriter} */ public class EdgeTreeReader implements ShapeTreeReader { - private static final Optional OPTIONAL_FALSE = Optional.of(false); - private static final Optional OPTIONAL_TRUE = Optional.of(true); - private static final Optional OPTIONAL_EMPTY = Optional.empty(); - private final ByteBufferStreamInput input; private final int startPosition; private final boolean hasArea; - private Extent treeExtent; public EdgeTreeReader(ByteBufferStreamInput input, boolean hasArea) throws IOException { this.startPosition = input.position(); this.input = input; this.hasArea = hasArea; - this.treeExtent = null; } public Extent getExtent() throws IOException { - if (treeExtent == null) { - resetInputPosition(); - treeExtent = new Extent(input); - } - return treeExtent; + resetInputPosition(); + return new Extent(input); } @Override @@ -68,51 +58,44 @@ public double getCentroidY() { * Returns true if the rectangle query and the edge tree's shape overlap */ @Override - public GeoRelation relate(Extent extent) throws IOException { - if (crosses(extent)) { - return GeoRelation.QUERY_CROSSES; - } else if (hasArea && containsBottomLeft(extent)){ - return GeoRelation.QUERY_INSIDE; + public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { + resetInputPosition(); + // check extent + int treeMaxY = input.readInt(); + int treeMinY = input.readInt(); + int negLeft = input.readInt(); + int negRight = input.readInt(); + int posLeft = input.readInt(); + int posRight = input.readInt(); + int treeMinX = Math.min(negLeft, posLeft); + int treeMaxX = Math.max(negRight, posRight); + if (treeMinY > maxY || treeMaxX < minX || treeMaxY < minY || treeMinX > maxX) { + return GeoRelation.QUERY_DISJOINT; // tree and bbox-query are disjoint } - return GeoRelation.QUERY_DISJOINT; - } - - static Optional checkExtent(Extent treeExtent, Extent extent) throws IOException { - if (treeExtent.minY() > extent.maxY() || treeExtent.maxX() < extent.minX() - || treeExtent.maxY() < extent.minY() || treeExtent.minX() > extent.maxX()) { - return OPTIONAL_FALSE; // tree and bbox-query are disjoint + if (minX <= treeMinX && minY <= treeMinY && maxX >= treeMaxX && maxY >= treeMaxY) { + return GeoRelation.QUERY_CROSSES; // bbox-query fully contains tree's } - if (extent.minX() <= treeExtent.minX() && extent.minY() <= treeExtent.minY() - && extent.maxX() >= treeExtent.maxX() && extent.maxY() >= treeExtent.maxY()) { - return OPTIONAL_TRUE; // bbox-query fully contains tree's extent. + if (crosses(treeMaxX, treeMaxY, minX, minY, maxX, maxY)) { + return GeoRelation.QUERY_CROSSES; + } else if (hasArea && containsBottomLeft(treeMaxX, treeMaxY, minX, minY, maxY)){ + return GeoRelation.QUERY_INSIDE; } - return OPTIONAL_EMPTY; + return GeoRelation.QUERY_DISJOINT; } - boolean containsBottomLeft(Extent extent) throws IOException { - Optional extentCheck = checkExtent(getExtent(), extent); - if (extentCheck.isPresent()) { - return extentCheck.get(); - } - + private boolean containsBottomLeft(int treeMaxX, int treeMaxY, int minX, int minY, int maxY) throws IOException { resetToRootEdge(); if (input.readBoolean()) { /* has edges */ - return containsBottomLeft(input.position(), extent); + return containsBottomLeft(input.position(), treeMaxX, treeMaxY, minX, minY, maxY); } return false; } - public boolean crosses(Extent extent) throws IOException { - resetInputPosition(); - Optional extentCheck = checkExtent(getExtent(), extent); - if (extentCheck.isPresent()) { - return extentCheck.get(); - } - + private boolean crosses(int treeMaxX, int treeMaxY, int minX, int minY, int maxX, int maxY) throws IOException { resetToRootEdge(); if (input.readBoolean()) { /* has edges */ - return crosses(input.position(), extent); + return crosses(input.position(), treeMaxX, treeMaxY, minX, minY, maxX, maxY); } return false; } @@ -121,15 +104,16 @@ public boolean crosses(Extent extent) throws IOException { * Returns true if the bottom-left point of the rectangle query is contained within the * tree's edges. */ - private boolean containsBottomLeft(int edgePosition, Extent extent) throws IOException { + private boolean containsBottomLeft(int edgePosition, int treeMaxX, int treeMaxY, + int minX, int minY, int maxY) throws IOException { // start read edge from bytes input.position(edgePosition); - int maxY = Math.toIntExact(treeExtent.maxY() - input.readVLong()); - int minY = Math.toIntExact(treeExtent.maxY() - input.readVLong()); - int x1 = Math.toIntExact(treeExtent.maxX() - input.readVLong()); - int x2 = Math.toIntExact(treeExtent.maxX() - input.readVLong()); - int y1 = Math.toIntExact(treeExtent.maxY() - input.readVLong()); - int y2 = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int thisMaxY = Math.toIntExact(treeMaxY - input.readVLong()); + int thisMinY = Math.toIntExact(treeMaxY - input.readVLong()); + int x1 = Math.toIntExact(treeMaxX - input.readVLong()); + int x2 = Math.toIntExact(treeMaxX - input.readVLong()); + int y1 = Math.toIntExact(treeMaxY - input.readVLong()); + int y2 = Math.toIntExact(treeMaxY - input.readVLong()); int rightOffset = input.readVInt(); if (rightOffset == 1) { rightOffset = 0; @@ -140,20 +124,19 @@ private boolean containsBottomLeft(int edgePosition, Extent extent) throws IOExc // end read edge from bytes boolean res = false; - if (maxY >= extent.minY()) { + if (thisMaxY >= minY) { // is bbox-query contained within linearRing // cast infinite ray to the right from bottom-left of bbox-query to see if it intersects edge - if (lineCrossesLineWithBoundary(x1, y1, x2, y2, extent.minX(), extent.minY(), Integer.MAX_VALUE, - extent.minY())) { + if (lineCrossesLineWithBoundary(x1, y1, x2, y2, minX, minY, Integer.MAX_VALUE, minY)) { res = true; } if (rightOffset > 0) { /* has left node */ - res ^= containsBottomLeft(streamOffset, extent); + res ^= containsBottomLeft(streamOffset, treeMaxX, treeMaxY, minX, minY, maxY); } - if (rightOffset >= 0 && extent.maxY() >= minY) { /* no right node if rightOffset == -1 */ - res ^= containsBottomLeft(streamOffset + rightOffset, extent); + if (rightOffset >= 0 && maxY >= thisMinY) { /* no right node if rightOffset == -1 */ + res ^= containsBottomLeft(streamOffset + rightOffset, treeMaxX, treeMaxY, minX, minY, maxY); } } return res; @@ -162,15 +145,15 @@ private boolean containsBottomLeft(int edgePosition, Extent extent) throws IOExc /** * Returns true if the box crosses any edge in this edge subtree * */ - private boolean crosses(int edgePosition, Extent extent) throws IOException { + private boolean crosses(int edgePosition, int treeMaxX, int treeMaxY, int minX, int minY, int maxX, int maxY) throws IOException { // start read edge from bytes input.position(edgePosition); - int maxY = Math.toIntExact(treeExtent.maxY() - input.readVLong()); - int minY = Math.toIntExact(treeExtent.maxY() - input.readVLong()); - int x1 = Math.toIntExact(treeExtent.maxX() - input.readVLong()); - int x2 = Math.toIntExact(treeExtent.maxX() - input.readVLong()); - int y1 = Math.toIntExact(treeExtent.maxY() - input.readVLong()); - int y2 = Math.toIntExact(treeExtent.maxY() - input.readVLong()); + int thisMaxY = Math.toIntExact(treeMaxY - input.readVLong()); + int thisMinY = Math.toIntExact(treeMaxY - input.readVLong()); + int x1 = Math.toIntExact(treeMaxX - input.readVLong()); + int x2 = Math.toIntExact(treeMaxX - input.readVLong()); + int y1 = Math.toIntExact(treeMaxY - input.readVLong()); + int y2 = Math.toIntExact(treeMaxY - input.readVLong()); int rightOffset = input.readVInt(); if (rightOffset == 1) { rightOffset = 0; @@ -181,37 +164,37 @@ private boolean crosses(int edgePosition, Extent extent) throws IOException { // end read edge from bytes // we just have to cross one edge to answer the question, so we descend the tree and return when we do. - if (maxY >= extent.minY()) { - boolean outside = (y1 < extent.minY() && y2 < extent.minY()) || - (y1 > extent.maxY() && y2 > extent.maxY()) || - (x1 < extent.minX() && x2 < extent.minX()) || - (x1 > extent.maxX() && x2 > extent.maxX()); + if (thisMaxY >= minY) { + boolean outside = (y1 < minY && y2 < minY) || + (y1 > maxY && y2 > maxY) || + (x1 < minX && x2 < minX) || + (x1 > maxX && x2 > maxX); // does rectangle's edges intersect or reside inside polygon's edge if (outside == false && (lineCrossesLineWithBoundary(x1, y1, x2, y2, - extent.minX(), extent.minY(), extent.maxX(), extent.minY()) || + minX, minY, maxX, minY) || lineCrossesLineWithBoundary(x1, y1, x2, y2, - extent.maxX(), extent.minY(), extent.maxX(), extent.maxY()) || + maxX, minY, maxX, maxY) || lineCrossesLineWithBoundary(x1, y1, x2, y2, - extent.maxX(), extent.maxY(), extent.minX(), extent.maxY()) || + maxX, maxY, minX, maxY) || lineCrossesLineWithBoundary(x1, y1, x2, y2, - extent.minX(), extent.maxY(), extent.minX(), extent.minY()))) { + minX, maxY, minX, minY))) { return true; } // does this edge fully reside within the rectangle's area - if (extent.minX() <= Math.min(x1, x2) && extent.minY() <= Math.min(y1, y2) - && extent.maxX() >= Math.max(x1, x2) && extent.maxY() >= Math.max(y1, y2)) { + if (minX <= Math.min(x1, x2) && minY <= Math.min(y1, y2) + && maxX >= Math.max(x1, x2) && maxY >= Math.max(y1, y2)) { return true; } /* has left node */ - if (rightOffset > 0 && crosses(streamOffset, extent)) { + if (rightOffset > 0 && crosses(streamOffset, treeMaxX, treeMaxY, minX, minY, maxX, maxY)) { return true; } /* no right node if rightOffset == -1 */ - if (rightOffset >= 0 && extent.maxY() >= minY && crosses(streamOffset + rightOffset, extent)) { + if (rightOffset >= 0 && maxY >= thisMinY && crosses(streamOffset + rightOffset, treeMaxX, treeMaxY, minX, minY, maxX, maxY)) { return true; } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java index 236ff568f1c43..39fcaa6a64619 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java @@ -24,7 +24,6 @@ import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Optional; /** * A reusable tree reader. @@ -82,14 +81,26 @@ public Extent getExtent() throws IOException { } @Override - public GeoRelation relate(Extent extent) throws IOException { + public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { GeoRelation relation = GeoRelation.QUERY_DISJOINT; input.position(startPosition + EXTENT_OFFSET); boolean hasExtent = input.readBoolean(); if (hasExtent) { - Optional extentCheck = EdgeTreeReader.checkExtent(new Extent(input), extent); - if (extentCheck.isPresent()) { - return extentCheck.get() ? GeoRelation.QUERY_INSIDE : GeoRelation.QUERY_DISJOINT; + int thisMaxY = input.readInt(); + int thisMinY = input.readInt(); + int negLeft = input.readInt(); + int negRight = input.readInt(); + int posLeft = input.readInt(); + int posRight = input.readInt(); + int thisMinX = Math.min(negLeft, posLeft); + int thisMaxX = Math.max(negRight, posRight); + + // check extent + if (thisMinY > maxY || thisMaxX < minX || thisMaxY < minY || thisMinX > maxX) { + return GeoRelation.QUERY_DISJOINT; // tree and bbox-query are disjoint + } + if (minX <= thisMinX && minY <= thisMinY && maxX >= thisMaxX && maxY >= thisMaxY) { + return GeoRelation.QUERY_CROSSES; // bbox-query fully contains tree's } } @@ -106,7 +117,7 @@ public GeoRelation relate(Extent extent) throws IOException { // read ShapeType ordinal to avoid readEnum allocations int shapeTypeOrdinal = input.readVInt(); ShapeTreeReader reader = getReader(shapeTypeOrdinal, coordinateEncoder, input); - GeoRelation shapeRelation = reader.relate(extent); + GeoRelation shapeRelation = reader.relate(minX, minY, maxX, maxY); if (GeoRelation.QUERY_CROSSES == shapeRelation || (GeoRelation.QUERY_DISJOINT == shapeRelation && GeoRelation.QUERY_INSIDE == relation) ) { diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java index 737386026a09a..707a85513b933 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java @@ -63,7 +63,7 @@ public double getCentroidY() { @Override - public GeoRelation relate(Extent extent) throws IOException { + public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { Deque stack = new ArrayDeque<>(); stack.push(0); stack.push(size - 1); @@ -78,7 +78,7 @@ public GeoRelation relate(Extent extent) throws IOException { // TODO serialize to re-usable array instead of serializing in each step int x = readX(i); int y = readY(i); - if (x >= extent.minX() && x <= extent.maxX() && y >= extent.minY() && y <= extent.maxY()) { + if (x >= minX && x <= maxX && y >= minY && y <= maxY) { return GeoRelation.QUERY_CROSSES; } } @@ -88,15 +88,15 @@ public GeoRelation relate(Extent extent) throws IOException { int middle = (right + left) >> 1; int x = readX(middle); int y = readY(middle); - if (x >= extent.minX() && x <= extent.maxX() && y >= extent.minY() && y <= extent.maxY()) { + if (x >= minX && x <= maxX && y >= minY && y <= maxY) { return GeoRelation.QUERY_CROSSES; } - if ((axis == 0 && extent.minX() <= x) || (axis == 1 && extent.minY() <= y)) { + if ((axis == 0 && minX <= x) || (axis == 1 && minY <= y)) { stack.push(left); stack.push(middle - 1); stack.push(1 - axis); } - if ((axis == 0 && extent.maxX() >= x) || (axis == 1 && extent.maxY() >= y)) { + if ((axis == 0 && maxX >= x) || (axis == 1 && maxY >= y)) { stack.push(middle + 1); stack.push(right); stack.push(1 - axis); diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java index 6474a17da5f0b..e330c0f8e1d4d 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java @@ -61,9 +61,9 @@ public double getCentroidY() { * Returns true if the rectangle query and the edge tree's shape overlap */ @Override - public GeoRelation relate(Extent extent) throws IOException { + public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { if (holes != null) { - GeoRelation relation = holes.relate(extent); + GeoRelation relation = holes.relate(minX, minY, maxX, maxY); if (GeoRelation.QUERY_CROSSES == relation) { return GeoRelation.QUERY_CROSSES; } @@ -71,6 +71,6 @@ public GeoRelation relate(Extent extent) throws IOException { return GeoRelation.QUERY_DISJOINT; } } - return outerShell.relate(extent); + return outerShell.relate(minX, minY, maxX, maxY); } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java index 9bb88980e2f0c..743ae712b4012 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java @@ -28,7 +28,7 @@ public interface ShapeTreeReader { Extent getExtent() throws IOException; - GeoRelation relate(Extent extent) throws IOException; + GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException; double getCentroidX() throws IOException; double getCentroidY() throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index 626234dac50a8..749bc8ff2714f 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -79,16 +79,12 @@ public double getCentroidY() throws IOException { return coordinateEncoder.decodeY(input.readInt()); } - @Override - public GeoRelation relate(Extent extent) throws IOException { - return relate(extent.minX(), extent.maxX(), extent.minY(), extent.maxY()); - } - /** * Compute the relation with the provided bounding box. If the result is CELL_INSIDE_QUERY * then the bounding box is within the shape. */ - private GeoRelation relate(int minX, int maxX, int minY, int maxY) throws IOException { + @Override + public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { input.position(extentOffset); int thisMaxX = input.readInt(); int thisMinX = Math.toIntExact(thisMaxX - input.readVLong()); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 78bc5bec75d19..78829cca7dbb9 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -151,9 +151,8 @@ public GeoRelation relate(Rectangle rectangle) { int maxX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMaxX()); int minY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMinY()); int maxY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMaxY()); - Extent extent = new Extent(maxY, minY, minX, maxX, minX, maxX); try { - return reader.relate(extent); + return reader.relate(minX, minY, maxX, maxY); } catch (IOException e) { throw new IllegalStateException("unable to check intersection", e); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java b/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java index 7e3652fe444c1..1162782c329d6 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java +++ b/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java @@ -326,8 +326,10 @@ private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) } private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { + + Extent bufferBounds = bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize); GeoRelation relation = geometryTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) - .relate(bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize)); + .relate(bufferBounds.minX(), bufferBounds.minY(), bufferBounds.maxX(), bufferBounds.maxY()); return relation == GeoRelation.QUERY_CROSSES || relation == GeoRelation.QUERY_INSIDE; } diff --git a/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java b/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java index 35df9c3bc3278..675620ecf8db6 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java @@ -171,11 +171,10 @@ private int getHashesAtLevel(ShapeTreeReader reader, CoordinateEncoder encoder, String[] hashes = GeohashUtils.getSubGeohashes(hash); for (int i =0; i < hashes.length; i++) { Rectangle r = Geohash.toBoundingBox(hashes[i]); - Extent extent = Extent.fromPoints(encoder.encodeX(r.getMinLon()), + GeoRelation rel = reader.relate(encoder.encodeX(r.getMinLon()), encoder.encodeY(r.getMinLat()), encoder.encodeX(r.getMaxLon()), encoder.encodeY(r.getMaxLat())); - GeoRelation rel = reader.relate(extent); if (rel == GeoRelation.QUERY_CROSSES) { if (hashes[i].length() == maxPrecision) { hits++; diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java index c1754b3e8409c..3b2da0e617c24 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -35,7 +35,7 @@ public class GeoTestUtils { public static void assertRelation(GeoRelation expectedRelation, ShapeTreeReader reader, Extent extent) throws IOException { - GeoRelation actualRelation = reader.relate(extent); + GeoRelation actualRelation = reader.relate(extent.minX(), extent.minY(), extent.maxX(), extent.maxY()); assertThat(actualRelation, equalTo(expectedRelation)); } From 0d4957fcb14d09ea1de3b77546de02bd2d586c28 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 5 Dec 2019 20:18:41 +0100 Subject: [PATCH 43/62] Add full extent to the triangle tree to support geo_bounds aggregation (#49847) --- .../org/elasticsearch/common/geo/Extent.java | 58 +++++++++++++++++-- .../common/geo/TriangleTreeReader.java | 36 ++++++++---- .../common/geo/TriangleTreeWriter.java | 56 +++++++++++------- .../index/mapper/GeoShapeIndexer.java | 4 ++ .../elasticsearch/common/geo/ExtentTests.java | 22 +++++++ 5 files changed, 137 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index 46991c8a2801f..155b391c1ff68 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -32,12 +32,21 @@ public class Extent implements Writeable { static final int WRITEABLE_SIZE_IN_BYTES = 24; - public final int top; - public final int bottom; - public final int negLeft; - public final int negRight; - public final int posLeft; - public final int posRight; + public int top; + public int bottom; + public int negLeft; + public int negRight; + public int posLeft; + public int posRight; + + public Extent() { + this.top = Integer.MIN_VALUE; + this.bottom = Integer.MAX_VALUE; + this.negLeft = Integer.MAX_VALUE; + this.negRight = Integer.MIN_VALUE; + this.posLeft = Integer.MAX_VALUE; + this.posRight = Integer.MIN_VALUE; + } public Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { this.top = top; @@ -52,6 +61,43 @@ public Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int p this(input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt()); } + public void reset(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + this.top = top; + this.bottom = bottom; + this.negLeft = negLeft; + this.negRight = negRight; + this.posLeft = posLeft; + this.posRight = posRight; + } + + /** + * Adds the extent of two points representing a bounding box's bottom-left + * and top-right points. The bounding box must not cross the dateline. + * + * @param bottomLeftX the bottom-left x-coordinate + * @param bottomLeftY the bottom-left y-coordinate + * @param topRightX the top-right x-coordinate + * @param topRightY the top-right y-coordinate + */ + public void addRectangle(int bottomLeftX, int bottomLeftY, int topRightX, int topRightY) { + assert bottomLeftX <= topRightX; + assert bottomLeftY <= topRightY; + this.bottom = Math.min(this.bottom, bottomLeftY); + this.top = Math.max(this.top, topRightY); + if (bottomLeftX < 0 && topRightX < 0) { + this.negLeft = Math.min(this.negLeft, bottomLeftX); + this.negRight = Math.max(this.negRight, topRightX); + } else if (bottomLeftX < 0) { + this.negLeft = Math.min(this.negLeft, bottomLeftX); + this.negRight = Math.max(this.negRight, bottomLeftX); + this.posLeft = Math.min(this.posLeft, topRightX); + this.posRight = Math.max(this.posRight, topRightX); + } else { + this.posLeft = Math.min(this.posLeft, bottomLeftX); + this.posRight = Math.max(this.posRight, topRightX); + } + } + /** * calculates the extent of a point, which is the point itself. * @param x the x-coordinate of the point diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index 749bc8ff2714f..8afae43b8da9c 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -39,26 +39,38 @@ public class TriangleTreeReader implements ShapeTreeReader { private ByteBufferStreamInput input; private final CoordinateEncoder coordinateEncoder; private final Rectangle2D rectangle2D; + private final Extent extent; + private int treeOffset; public TriangleTreeReader(CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; this.rectangle2D = new Rectangle2D(); + this.extent = new Extent(); } - public void reset(BytesRef bytesRef) { + public void reset(BytesRef bytesRef) throws IOException { this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + treeOffset = 0; } /** * returns the bounding box of the geometry in the format [minX, maxX, minY, maxY]. */ public Extent getExtent() throws IOException { - input.position(extentOffset); - int thisMaxX = input.readInt(); - int thisMinX = Math.toIntExact(thisMaxX - input.readVLong()); - int thisMaxY = input.readInt(); - int thisMinY = Math.toIntExact(thisMaxY - input.readVLong()); - return Extent.fromPoints(thisMinX, thisMinY, thisMaxX, thisMaxY); + if (treeOffset == 0) { + input.position(extentOffset); + int top = input.readInt(); + int bottom = Math.toIntExact(top - input.readVLong()); + int posRight = input.readInt(); + int posLeft = input.readInt(); + int negRight = input.readInt(); + int negLeft = input.readInt(); + extent.reset(top, bottom, negLeft, negRight, posLeft, posRight); + treeOffset = input.position(); + } else { + input.position(treeOffset); + } + return extent; } /** @@ -85,11 +97,11 @@ public double getCentroidY() throws IOException { */ @Override public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { - input.position(extentOffset); - int thisMaxX = input.readInt(); - int thisMinX = Math.toIntExact(thisMaxX - input.readVLong()); - int thisMaxY = input.readInt(); - int thisMinY = Math.toIntExact(thisMaxY - input.readVLong()); + Extent extent = getExtent(); + int thisMaxX = extent.maxX(); + int thisMinX = extent.minX(); + int thisMaxY = extent.maxY(); + int thisMinY = extent.minY(); if (minX <= thisMinX && maxX >= thisMaxX && minY <= thisMinY && maxY >= thisMaxY) { // the rectangle fully contains the shape return GeoRelation.QUERY_CROSSES; diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java index 95e58bc056c7e..f1cca92c303f3 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java @@ -55,10 +55,12 @@ public class TriangleTreeWriter extends ShapeTreeWriter { private final CoordinateEncoder coordinateEncoder; private final CentroidCalculator centroidCalculator; private final ShapeType type; + private Extent extent; public TriangleTreeWriter(Geometry geometry, CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; this.centroidCalculator = new CentroidCalculator(); + this.extent = new Extent(); this.type = geometry.type(); TriangleTreeBuilder builder = new TriangleTreeBuilder(coordinateEncoder); geometry.visit(builder); @@ -69,13 +71,18 @@ public TriangleTreeWriter(Geometry geometry, CoordinateEncoder coordinateEncoder public void writeTo(StreamOutput out) throws IOException { out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); + out.writeInt(extent.top); + out.writeVLong((long) extent.top - extent.bottom); + out.writeInt(extent.posRight); + out.writeInt(extent.posLeft); + out.writeInt(extent.negRight); + out.writeInt(extent.negLeft); node.writeTo(out); } @Override public Extent getExtent() { - // do it right - return Extent.fromPoints(node.minX, node.minY, node.maxX, node.maxY); + return extent; } @Override @@ -93,9 +100,10 @@ public CentroidCalculator getCentroidCalculator() { */ class TriangleTreeBuilder implements GeometryVisitor { - private List triangles; + private final List triangles; private final CoordinateEncoder coordinateEncoder; + TriangleTreeBuilder(CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; this.triangles = new ArrayList<>(); @@ -115,10 +123,12 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - for (int i =0; i < line.length(); i++) { + for (int i = 0; i < line.length(); i++) { centroidCalculator.addCoordinate(line.getX(i), line.getY(i)); } - addTriangles(TriangleTreeLeaf.fromLine(coordinateEncoder, line)); + org.apache.lucene.geo.Line luceneLine = GeoShapeIndexer.toLuceneLine(line); + addToExtent(luceneLine.minLon, luceneLine.maxLon, luceneLine.minLat, luceneLine.maxLat); + addTriangles(TriangleTreeLeaf.fromLine(coordinateEncoder, luceneLine)); return null; } @@ -136,7 +146,9 @@ public Void visit(Polygon polygon) { for (int i =0; i < polygon.getPolygon().length() - 1; i++) { centroidCalculator.addCoordinate(polygon.getPolygon().getX(i), polygon.getPolygon().getY(i)); } - addTriangles(TriangleTreeLeaf.fromPolygon(coordinateEncoder, polygon)); + org.apache.lucene.geo.Polygon lucenePolygon = GeoShapeIndexer.toLucenePolygon(polygon); + addToExtent(lucenePolygon.minLon, lucenePolygon.maxLon, lucenePolygon.minLat, lucenePolygon.maxLat); + addTriangles(TriangleTreeLeaf.fromPolygon(coordinateEncoder, lucenePolygon)); return null; } @@ -152,6 +164,7 @@ public Void visit(MultiPolygon multiPolygon) { public Void visit(Rectangle r) { centroidCalculator.addCoordinate(r.getMinX(), r.getMinY()); centroidCalculator.addCoordinate(r.getMaxX(), r.getMaxY()); + addToExtent(r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); addTriangles(TriangleTreeLeaf.fromRectangle(coordinateEncoder, r)); return null; } @@ -159,6 +172,7 @@ public Void visit(Rectangle r) { @Override public Void visit(Point point) { centroidCalculator.addCoordinate(point.getX(), point.getY()); + addToExtent(point.getLon(), point.getLon(), point.getLat(), point.getLat()); addTriangles(TriangleTreeLeaf.fromPoints(coordinateEncoder, point)); return null; } @@ -181,6 +195,14 @@ public Void visit(Circle circle) { throw new IllegalArgumentException("invalid shape type found [Circle]"); } + private void addToExtent(double minLon, double maxLon, double minLat, double maxLat) { + int minX = coordinateEncoder.encodeX(minLon); + int maxX = coordinateEncoder.encodeX(maxLon); + int minY = coordinateEncoder.encodeY(minLat); + int maxY = coordinateEncoder.encodeY(maxLat); + extent.addRectangle(minX, minY, maxX, maxY); + } + public TriangleTreeNode build() { if (triangles.size() == 1) { @@ -191,10 +213,6 @@ public TriangleTreeNode build() { nodes[i] = new TriangleTreeNode(triangles.get(i)); } TriangleTreeNode root = createTree(nodes, 0, triangles.size() - 1, true); - for (TriangleTreeNode node : nodes) { - root.minX = Math.min(root.minX, node.minX); - root.minY = Math.min(root.minY, node.minY); - } return root; } @@ -274,10 +292,6 @@ protected TriangleTreeNode(TriangleTreeLeaf component) { @Override public void writeTo(StreamOutput out) throws IOException { BytesStreamOutput scratchBuffer = new BytesStreamOutput(); - out.writeInt(maxX); - out.writeVLong((long) maxX - minX); - out.writeInt(maxY); - out.writeVLong((long) maxY - minY); writeMetadata(out); writeComponent(out); if (left != null) { @@ -593,19 +607,19 @@ private static List fromRectangle(CoordinateEncoder encoder, R return triangles; } - private static List fromLine(CoordinateEncoder encoder, Line line) { - List triangles = new ArrayList<>(line.length() - 1); - for (int i = 0, j = 1; i < line.length() - 1; i++, j++) { - triangles.add(new TriangleTreeLeaf(encoder.encodeX(line.getX(i)), encoder.encodeY(line.getY(i)), - encoder.encodeX(line.getX(j)), encoder.encodeY(line.getY(j)))); + private static List fromLine(CoordinateEncoder encoder, org.apache.lucene.geo.Line line) { + List triangles = new ArrayList<>(line.numPoints() - 1); + for (int i = 0, j = 1; i < line.numPoints() - 1; i++, j++) { + triangles.add(new TriangleTreeLeaf(encoder.encodeX(line.getLon(i)), encoder.encodeY(line.getLat(i)), + encoder.encodeX(line.getLon(j)), encoder.encodeY(line.getLat(j)))); } return triangles; } - private static List fromPolygon(CoordinateEncoder encoder, Polygon polygon) { + private static List fromPolygon(CoordinateEncoder encoder, org.apache.lucene.geo.Polygon polygon) { // TODO: We are going to be tessellating the polygon twice, can we do something? // TODO: Tessellator seems to have some reference to the encoding but does not need to have. - List tessellation = Tessellator.tessellate(GeoShapeIndexer.toLucenePolygon(polygon)); + List tessellation = Tessellator.tessellate(polygon); List triangles = new ArrayList<>(tessellation.size()); for (Tessellator.Triangle t : tessellation) { triangles.add(new TriangleTreeLeaf(encoder.encodeX(t.getX(0)), encoder.encodeY(t.getY(0)), t.isEdgefromPolygon(0), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java index ece0561e06168..4b8db174ca7df 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java @@ -1084,6 +1084,10 @@ public static org.apache.lucene.geo.Polygon toLucenePolygon(Polygon polygon) { return new org.apache.lucene.geo.Polygon(polygon.getPolygon().getY(), polygon.getPolygon().getX(), holes); } + public static org.apache.lucene.geo.Line toLuceneLine(Line line) { + return new org.apache.lucene.geo.Line(line.getLats(), line.getLons()); + } + /** * Normalizes longitude while accepting -180 degrees as a valid value */ diff --git a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java index 2c527845240c2..378fe9e3f054f 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java @@ -68,6 +68,28 @@ public void testFromPoints() { } } + public void testAddRectangle() { + Extent extent = new Extent(); + int bottomLeftX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(-175); + int bottomLeftY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(-10); + int topRightX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(-170); + int topRightY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(10); + extent.addRectangle(bottomLeftX, bottomLeftY, topRightX, topRightY); + assertThat(extent.minX(), equalTo(bottomLeftX)); + assertThat(extent.maxX(), equalTo(topRightX)); + assertThat(extent.minY(), equalTo(bottomLeftY)); + assertThat(extent.maxY(), equalTo(topRightY)); + int bottomLeftX2 = GeoShapeCoordinateEncoder.INSTANCE.encodeX(170); + int bottomLeftY2 = GeoShapeCoordinateEncoder.INSTANCE.encodeY(-20); + int topRightX2 = GeoShapeCoordinateEncoder.INSTANCE.encodeX(175); + int topRightY2 = GeoShapeCoordinateEncoder.INSTANCE.encodeY(20); + extent.addRectangle(bottomLeftX2, bottomLeftY2, topRightX2, topRightY2); + assertThat(extent.minX(), equalTo(bottomLeftX)); + assertThat(extent.maxX(), equalTo(topRightX2)); + assertThat(extent.minY(), equalTo(bottomLeftY2)); + assertThat(extent.maxY(), equalTo(topRightY2)); + } + @Override protected Extent createTestInstance() { return new Extent(randomIntBetween(-10, 10), randomIntBetween(-10, 10), randomIntBetween(-10, 10), From a4402b3a00c6498819958bfd5335e4d3e44f402f Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 9 Dec 2019 16:13:32 -0800 Subject: [PATCH 44/62] Use TriangleTree for GeoShape Doc Values (#49840) After lots of evaluation and benchmarking, it seems the TriangleTree is preferred over the GeometryTree for the following reasons - simpler to model all shapes as one tree instead of specializing and optimizing for edge-cases. (GeometryCollection of Points is stored the same as a MultiPoint) - Although there are more situations where the EdgeTree out-performs the TriangleTree, the times it is faster it is faster by a lot because it is these times that the EdgeTree must traverse O(n) of the edges to determine a queried tile is outside of the shape. https://gist.github.com/talevy/f06ef43be1e97afb1ee53f25b980a4a0 Downsides of the Triangle when compared to Geometry Tree - it is not possible to reverse-engineer into the original geometry for use in scripting - Points cannot be stored as compactly as in the GeometryTree - not faster on every single relate query --- .../common/geo/CentroidCalculator.java | 2 +- .../common/geo/EdgeTreeReader.java | 211 ---------- .../common/geo/EdgeTreeWriter.java | 259 ------------- .../org/elasticsearch/common/geo/Extent.java | 3 +- .../common/geo/GeometryTreeReader.java | 146 ------- .../common/geo/GeometryTreeWriter.java | 226 ----------- .../common/geo/Point2DReader.java | 120 ------ .../common/geo/Point2DWriter.java | 201 ---------- .../common/geo/PolygonTreeReader.java | 76 ---- .../common/geo/PolygonTreeWriter.java | 68 ---- .../common/geo/ShapeTreeReader.java | 34 -- .../common/geo/TriangleTreeReader.java | 5 +- .../index/fielddata/MultiGeoValues.java | 13 +- .../plain/LatLonShapeDVAtomicFieldData.java | 4 +- .../mapper/BinaryGeoShapeDocValuesField.java | 4 +- .../common/geo/AbstractTreeTestCase.java | 362 ------------------ .../common/geo/EdgeTreeTests.java | 184 --------- .../common/geo/EncodingComparisonTests.java | 207 ---------- .../common/geo/GeoTestUtils.java | 8 +- .../common/geo/GeometryTreeTests.java | 38 -- .../common/geo/Point2DTests.java | 85 ---- .../common/geo/TriangleTreeTests.java | 288 +++++++++++++- .../bucket/geogrid/GeoGridTilerTests.java | 32 +- .../aggregations/metrics/GeoBoundsIT.java | 26 +- .../support/MissingValuesTests.java | 17 +- 25 files changed, 331 insertions(+), 2288 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java delete mode 100644 server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java delete mode 100644 server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java delete mode 100644 server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java delete mode 100644 server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java delete mode 100644 server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java index c15c24d65ec40..998f2753420ff 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java +++ b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java @@ -21,7 +21,7 @@ /** * This class keeps a running Kahan-sum of coordinates - * that are to be averaged in {@link GeometryTreeWriter} for use + * that are to be averaged in {@link TriangleTreeWriter} for use * as the centroid of a shape. */ public class CentroidCalculator { diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java deleted file mode 100644 index b212f29cf8632..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeReader.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; - -import java.io.IOException; - -import static org.apache.lucene.geo.GeoUtils.lineCrossesLineWithBoundary; - -/** - * This {@link ShapeTreeReader} understands how to parse polygons - * serialized with the {@link EdgeTreeWriter} - */ -public class EdgeTreeReader implements ShapeTreeReader { - private final ByteBufferStreamInput input; - private final int startPosition; - private final boolean hasArea; - - public EdgeTreeReader(ByteBufferStreamInput input, boolean hasArea) throws IOException { - this.startPosition = input.position(); - this.input = input; - this.hasArea = hasArea; - } - - public Extent getExtent() throws IOException { - resetInputPosition(); - return new Extent(input); - } - - @Override - public double getCentroidX() { - throw new UnsupportedOperationException(); - } - - @Override - public double getCentroidY() { - throw new UnsupportedOperationException(); - } - - /** - * Returns true if the rectangle query and the edge tree's shape overlap - */ - @Override - public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { - resetInputPosition(); - // check extent - int treeMaxY = input.readInt(); - int treeMinY = input.readInt(); - int negLeft = input.readInt(); - int negRight = input.readInt(); - int posLeft = input.readInt(); - int posRight = input.readInt(); - int treeMinX = Math.min(negLeft, posLeft); - int treeMaxX = Math.max(negRight, posRight); - if (treeMinY > maxY || treeMaxX < minX || treeMaxY < minY || treeMinX > maxX) { - return GeoRelation.QUERY_DISJOINT; // tree and bbox-query are disjoint - } - if (minX <= treeMinX && minY <= treeMinY && maxX >= treeMaxX && maxY >= treeMaxY) { - return GeoRelation.QUERY_CROSSES; // bbox-query fully contains tree's - } - - if (crosses(treeMaxX, treeMaxY, minX, minY, maxX, maxY)) { - return GeoRelation.QUERY_CROSSES; - } else if (hasArea && containsBottomLeft(treeMaxX, treeMaxY, minX, minY, maxY)){ - return GeoRelation.QUERY_INSIDE; - } - return GeoRelation.QUERY_DISJOINT; - } - - private boolean containsBottomLeft(int treeMaxX, int treeMaxY, int minX, int minY, int maxY) throws IOException { - resetToRootEdge(); - if (input.readBoolean()) { /* has edges */ - return containsBottomLeft(input.position(), treeMaxX, treeMaxY, minX, minY, maxY); - } - return false; - } - - private boolean crosses(int treeMaxX, int treeMaxY, int minX, int minY, int maxX, int maxY) throws IOException { - resetToRootEdge(); - if (input.readBoolean()) { /* has edges */ - return crosses(input.position(), treeMaxX, treeMaxY, minX, minY, maxX, maxY); - } - return false; - } - - /** - * Returns true if the bottom-left point of the rectangle query is contained within the - * tree's edges. - */ - private boolean containsBottomLeft(int edgePosition, int treeMaxX, int treeMaxY, - int minX, int minY, int maxY) throws IOException { - // start read edge from bytes - input.position(edgePosition); - int thisMaxY = Math.toIntExact(treeMaxY - input.readVLong()); - int thisMinY = Math.toIntExact(treeMaxY - input.readVLong()); - int x1 = Math.toIntExact(treeMaxX - input.readVLong()); - int x2 = Math.toIntExact(treeMaxX - input.readVLong()); - int y1 = Math.toIntExact(treeMaxY - input.readVLong()); - int y2 = Math.toIntExact(treeMaxY - input.readVLong()); - int rightOffset = input.readVInt(); - if (rightOffset == 1) { - rightOffset = 0; - } else if (rightOffset == 0) { - rightOffset = -1; - } - int streamOffset = input.position(); - // end read edge from bytes - - boolean res = false; - if (thisMaxY >= minY) { - // is bbox-query contained within linearRing - // cast infinite ray to the right from bottom-left of bbox-query to see if it intersects edge - if (lineCrossesLineWithBoundary(x1, y1, x2, y2, minX, minY, Integer.MAX_VALUE, minY)) { - res = true; - } - - if (rightOffset > 0) { /* has left node */ - res ^= containsBottomLeft(streamOffset, treeMaxX, treeMaxY, minX, minY, maxY); - } - - if (rightOffset >= 0 && maxY >= thisMinY) { /* no right node if rightOffset == -1 */ - res ^= containsBottomLeft(streamOffset + rightOffset, treeMaxX, treeMaxY, minX, minY, maxY); - } - } - return res; - } - - /** - * Returns true if the box crosses any edge in this edge subtree - * */ - private boolean crosses(int edgePosition, int treeMaxX, int treeMaxY, int minX, int minY, int maxX, int maxY) throws IOException { - // start read edge from bytes - input.position(edgePosition); - int thisMaxY = Math.toIntExact(treeMaxY - input.readVLong()); - int thisMinY = Math.toIntExact(treeMaxY - input.readVLong()); - int x1 = Math.toIntExact(treeMaxX - input.readVLong()); - int x2 = Math.toIntExact(treeMaxX - input.readVLong()); - int y1 = Math.toIntExact(treeMaxY - input.readVLong()); - int y2 = Math.toIntExact(treeMaxY - input.readVLong()); - int rightOffset = input.readVInt(); - if (rightOffset == 1) { - rightOffset = 0; - } else if (rightOffset == 0) { - rightOffset = -1; - } - int streamOffset = input.position(); - // end read edge from bytes - - // we just have to cross one edge to answer the question, so we descend the tree and return when we do. - if (thisMaxY >= minY) { - boolean outside = (y1 < minY && y2 < minY) || - (y1 > maxY && y2 > maxY) || - (x1 < minX && x2 < minX) || - (x1 > maxX && x2 > maxX); - - // does rectangle's edges intersect or reside inside polygon's edge - if (outside == false && (lineCrossesLineWithBoundary(x1, y1, x2, y2, - minX, minY, maxX, minY) || - lineCrossesLineWithBoundary(x1, y1, x2, y2, - maxX, minY, maxX, maxY) || - lineCrossesLineWithBoundary(x1, y1, x2, y2, - maxX, maxY, minX, maxY) || - lineCrossesLineWithBoundary(x1, y1, x2, y2, - minX, maxY, minX, minY))) { - return true; - } - - // does this edge fully reside within the rectangle's area - if (minX <= Math.min(x1, x2) && minY <= Math.min(y1, y2) - && maxX >= Math.max(x1, x2) && maxY >= Math.max(y1, y2)) { - return true; - } - - /* has left node */ - if (rightOffset > 0 && crosses(streamOffset, treeMaxX, treeMaxY, minX, minY, maxX, maxY)) { - return true; - } - - /* no right node if rightOffset == -1 */ - if (rightOffset >= 0 && maxY >= thisMinY && crosses(streamOffset + rightOffset, treeMaxX, treeMaxY, minX, minY, maxX, maxY)) { - return true; - } - } - return false; - } - - private void resetInputPosition() throws IOException { - input.position(startPosition); - } - - private void resetToRootEdge() throws IOException { - input.position(startPosition + Extent.WRITEABLE_SIZE_IN_BYTES); // skip extent - } -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java deleted file mode 100644 index e897ce40732b8..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/EdgeTreeWriter.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.geometry.ShapeType; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Shape edge-tree writer for use in doc-values - */ -public class EdgeTreeWriter extends ShapeTreeWriter { - - private final Extent extent; - private final int numShapes; - private final CentroidCalculator centroidCalculator; - final Edge tree; - - - /** - * @param x array of the x-coordinate of points. - * @param y array of the y-coordinate of points. - * @param coordinateEncoder class that encodes from real-valued x/y to serialized integer coordinate values. - * @param hasArea whether the tree represents a Polygon that has a defined area - */ - EdgeTreeWriter(double[] x, double[] y, CoordinateEncoder coordinateEncoder, boolean hasArea) { - this(Collections.singletonList(x), Collections.singletonList(y), coordinateEncoder, hasArea); - } - - EdgeTreeWriter(List x, List y, CoordinateEncoder coordinateEncoder, boolean hasArea) { - this.centroidCalculator = new CentroidCalculator(); - this.numShapes = x.size(); - double top = Double.NEGATIVE_INFINITY; - double bottom = Double.POSITIVE_INFINITY; - double negLeft = Double.POSITIVE_INFINITY; - double negRight = Double.NEGATIVE_INFINITY; - double posLeft = Double.POSITIVE_INFINITY; - double posRight = Double.NEGATIVE_INFINITY; - - List edges = new ArrayList<>(); - for (int i = 0; i < y.size(); i++) { - for (int j = 1; j < y.get(i).length; j++) { - double y1 = y.get(i)[j - 1]; - double x1 = x.get(i)[j - 1]; - double y2 = y.get(i)[j]; - double x2 = x.get(i)[j]; - double edgeMinY, edgeMaxY; - if (y1 < y2) { - edgeMinY = y1; - edgeMaxY = y2; - } else { - edgeMinY = y2; - edgeMaxY = y1; - } - edges.add(new Edge(coordinateEncoder.encodeX(x1), coordinateEncoder.encodeY(y1), - coordinateEncoder.encodeX(x2), coordinateEncoder.encodeY(y2), - coordinateEncoder.encodeY(edgeMinY), coordinateEncoder.encodeY(edgeMaxY))); - - top = Math.max(top, Math.max(y1, y2)); - bottom = Math.min(bottom, Math.min(y1, y2)); - - // check first - if (x1 >= 0 && x1 < posLeft) { - posLeft = x1; - } - if (x1 >= 0 && x1 > posRight) { - posRight = x1; - } - if (x1 < 0 && x1 < negLeft) { - negLeft = x1; - } - if (x1 < 0 && x1 > negRight) { - negRight = x1; - } - - // check second - if (x2 >= 0 && x2 < posLeft) { - posLeft = x2; - } - if (x2 >= 0 && x2 > posRight) { - posRight = x2; - } - if (x2 < 0 && x2 < negLeft) { - negLeft = x2; - } - if (x2 < 0 && x2 > negRight) { - negRight = x2; - } - - // calculate centroid - centroidCalculator.addCoordinate(x1, y1); - if (j == y.get(i).length - 1 && hasArea == false) { - centroidCalculator.addCoordinate(x2, y2); - } - } - } - edges.sort(Edge::compareTo); - this.extent = new Extent(coordinateEncoder.encodeY(top), coordinateEncoder.encodeY(bottom), - coordinateEncoder.encodeX(negLeft), coordinateEncoder.encodeX(negRight), - coordinateEncoder.encodeX(posLeft), coordinateEncoder.encodeX(posRight)); - this.tree = createTree(edges, 0, edges.size() - 1); - } - - @Override - public Extent getExtent() { - return extent; - } - - @Override - public ShapeType getShapeType() { - return numShapes > 1 ? ShapeType.MULTILINESTRING: ShapeType.LINESTRING; - } - - @Override - public CentroidCalculator getCentroidCalculator() { - return centroidCalculator; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - extent.writeTo(out); - if (tree != null) { - out.writeBoolean(true); - tree.writeTo(out, new BytesStreamOutput(), extent); - } else { - out.writeBoolean(false); - } - } - - private static Edge createTree(List edges, int low, int high) { - if (low > high) { - return null; - } - // add midpoint - int mid = (low + high) >>> 1; - Edge newNode = edges.get(mid); - newNode.size = 1; - // add children - newNode.left = createTree(edges, low, mid - 1); - newNode.right = createTree(edges, mid + 1, high); - // pull up max values to this node - // and node count - if (newNode.left != null) { - newNode.maxY = Math.max(newNode.maxY, newNode.left.maxY); - newNode.size += newNode.left.size; - } - if (newNode.right != null) { - newNode.maxY = Math.max(newNode.maxY, newNode.right.maxY); - newNode.size += newNode.right.size; - } - return newNode; - } - - /** - * Object representing an in-memory edge-tree to be serialized - */ - static class Edge implements Comparable { - final int x1; - final int y1; - final int x2; - final int y2; - int minY; - int maxY; - int size; - Edge left; - Edge right; - - Edge(int x1, int y1, int x2, int y2, int minY, int maxY) { - this.x1 = x1; - this.y1 = y1; - this.x2 = x2; - this.y2 = y2; - this.minY = minY; - this.maxY = maxY; - } - - @Override - public int compareTo(Edge other) { - int ret = Integer.compare(minY, other.minY); - if (ret == 0) { - ret = Integer.compare(maxY, other.maxY); - } - return ret; - } - - private int writeEdgeContent(StreamOutput out, Extent extent) throws IOException { - long startPosition = out.position(); - out.writeVLong((long) extent.maxY() - maxY); - out.writeVLong((long) extent.maxY() - minY); - out.writeVLong((long) extent.maxX() - x1); - out.writeVLong((long) extent.maxX() - x2); - out.writeVLong((long) extent.maxY() - y1); - out.writeVLong((long) extent.maxY() - y2); - return Math.toIntExact(out.position() - startPosition); - } - - private void writeTo(StreamOutput out, BytesStreamOutput scratchBuffer, Extent extent) throws IOException { - writeEdgeContent(out, extent); - // left node is next node, write offset of right node - if (left != null) { - out.writeVInt(left.size(scratchBuffer, extent)); - } else if (right == null){ - out.writeVInt(0); - } else { - out.writeVInt(1); - } - if (left != null) { - left.writeTo(out, scratchBuffer, extent); - } - if (right != null) { - right.writeTo(out, scratchBuffer, extent); - } - } - - private int size(BytesStreamOutput scratchBuffer, Extent extent) throws IOException { - int size = writeEdgeContent(scratchBuffer, extent); - scratchBuffer.reset(); - // left node is next node, write offset of right node - if (left != null) { - int leftSize = left.size(scratchBuffer, extent); - scratchBuffer.reset(); - scratchBuffer.writeVInt(leftSize); - } else if (right == null){ - scratchBuffer.writeVInt(0); - } else { - scratchBuffer.writeVInt(1); - } - size += scratchBuffer.size(); - if (left != null) { - size += left.size(scratchBuffer, extent); - } - if (right != null) { - size += right.size(scratchBuffer, extent); - } - return size; - } - } -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index 155b391c1ff68..2c3eef24367a5 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -26,8 +26,7 @@ import java.util.Objects; /** - * Object representing the extent of a geometry object within a - * {@link GeometryTreeWriter} and {@link EdgeTreeWriter}. + * Object representing the extent of a geometry object within a {@link ShapeTreeWriter}. */ public class Extent implements Writeable { static final int WRITEABLE_SIZE_IN_BYTES = 24; diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java deleted file mode 100644 index 39fcaa6a64619..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeReader.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; -import org.elasticsearch.geometry.ShapeType; - -import java.io.IOException; -import java.nio.ByteBuffer; - -/** - * A reusable tree reader. - * - * This class supports checking bounding box - * relations against the serialized geometry tree. - */ -public class GeometryTreeReader implements ShapeTreeReader { - - private static final int EXTENT_OFFSET = 8; - private int startPosition; - private ByteBufferStreamInput input; - private final CoordinateEncoder coordinateEncoder; - - public GeometryTreeReader(CoordinateEncoder coordinateEncoder) { - this.coordinateEncoder = coordinateEncoder; - } - - private GeometryTreeReader(ByteBufferStreamInput input, CoordinateEncoder coordinateEncoder) throws IOException { - this.input = input; - startPosition = input.position(); - this.coordinateEncoder = coordinateEncoder; - } - - public void reset(BytesRef bytesRef) { - this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); - this.startPosition = 0; - } - - @Override - public double getCentroidX() throws IOException { - input.position(startPosition); - return coordinateEncoder.decodeX(input.readInt()); - } - - @Override - public double getCentroidY() throws IOException { - input.position(startPosition + 4); - return coordinateEncoder.decodeY(input.readInt()); - } - - @Override - public Extent getExtent() throws IOException { - input.position(startPosition + EXTENT_OFFSET); - Extent extent = input.readOptionalWriteable(Extent::new); - if (extent != null) { - return extent; - } - int numShapes = input.readVInt(); - assert numShapes == 1; - // read ShapeType ordinal to avoid readEnum allocations - int shapeTypeOrdinal = input.readVInt(); - ShapeTreeReader reader = getReader(shapeTypeOrdinal, coordinateEncoder, input); - return reader.getExtent(); - } - - @Override - public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { - GeoRelation relation = GeoRelation.QUERY_DISJOINT; - input.position(startPosition + EXTENT_OFFSET); - boolean hasExtent = input.readBoolean(); - if (hasExtent) { - int thisMaxY = input.readInt(); - int thisMinY = input.readInt(); - int negLeft = input.readInt(); - int negRight = input.readInt(); - int posLeft = input.readInt(); - int posRight = input.readInt(); - int thisMinX = Math.min(negLeft, posLeft); - int thisMaxX = Math.max(negRight, posRight); - - // check extent - if (thisMinY > maxY || thisMaxX < minX || thisMaxY < minY || thisMinX > maxX) { - return GeoRelation.QUERY_DISJOINT; // tree and bbox-query are disjoint - } - if (minX <= thisMinX && minY <= thisMinY && maxX >= thisMaxX && maxY >= thisMaxY) { - return GeoRelation.QUERY_CROSSES; // bbox-query fully contains tree's - } - } - - int numTrees = input.readVInt(); - int nextPosition = input.position(); - for (int i = 0; i < numTrees; i++) { - if (numTrees > 1) { - if (i > 0) { - input.position(nextPosition); - } - int pos = input.readVInt(); - nextPosition = input.position() + pos; - } - // read ShapeType ordinal to avoid readEnum allocations - int shapeTypeOrdinal = input.readVInt(); - ShapeTreeReader reader = getReader(shapeTypeOrdinal, coordinateEncoder, input); - GeoRelation shapeRelation = reader.relate(minX, minY, maxX, maxY); - if (GeoRelation.QUERY_CROSSES == shapeRelation || - (GeoRelation.QUERY_DISJOINT == shapeRelation && GeoRelation.QUERY_INSIDE == relation) - ) { - return GeoRelation.QUERY_CROSSES; - } else { - relation = shapeRelation; - } - } - - return relation; - } - - private static ShapeTreeReader getReader(int shapeTypeOrdinal, CoordinateEncoder coordinateEncoder, ByteBufferStreamInput input) - throws IOException { - if (shapeTypeOrdinal == ShapeType.POLYGON.ordinal()) { - return new PolygonTreeReader(input); - } else if (shapeTypeOrdinal == ShapeType.POINT.ordinal() || shapeTypeOrdinal == ShapeType.MULTIPOINT.ordinal()) { - return new Point2DReader(input); - } else if (shapeTypeOrdinal == ShapeType.LINESTRING.ordinal() || shapeTypeOrdinal == ShapeType.MULTILINESTRING.ordinal()) { - return new EdgeTreeReader(input, false); - } else if (shapeTypeOrdinal == ShapeType.GEOMETRYCOLLECTION.ordinal()) { - return new GeometryTreeReader(input, coordinateEncoder); - } - throw new UnsupportedOperationException("unsupported shape type ordinal [" + shapeTypeOrdinal + "]"); - } -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java deleted file mode 100644 index 08222031516a2..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/GeometryTreeWriter.java +++ /dev/null @@ -1,226 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.geometry.Circle; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.GeometryCollection; -import org.elasticsearch.geometry.GeometryVisitor; -import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; -import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.ShapeType; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * This is a tree-writer that serializes the - * appropriate tree structure for each type of - * {@link Geometry} into a byte array. - */ -public class GeometryTreeWriter extends ShapeTreeWriter { - - private final GeometryTreeBuilder builder; - private final CoordinateEncoder coordinateEncoder; - private CentroidCalculator centroidCalculator; - - public GeometryTreeWriter(Geometry geometry, CoordinateEncoder coordinateEncoder) { - this.coordinateEncoder = coordinateEncoder; - this.centroidCalculator = new CentroidCalculator(); - builder = new GeometryTreeBuilder(coordinateEncoder); - if (geometry.type() == ShapeType.GEOMETRYCOLLECTION) { - for (Geometry shape : (GeometryCollection) geometry) { - shape.visit(builder); - } - } else { - geometry.visit(builder); - } - } - - @Override - public Extent getExtent() { - return new Extent(builder.top, builder.bottom, builder.negLeft, builder.negRight, builder.posLeft, builder.posRight); - } - - @Override - public ShapeType getShapeType() { - return ShapeType.GEOMETRYCOLLECTION; - } - - @Override - public CentroidCalculator getCentroidCalculator() { - return centroidCalculator; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - // only write a geometry extent for the tree if the tree - // contains multiple sub-shapes - boolean multiShape = builder.shapeWriters.size() > 1; - Extent extent = null; - out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); - out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); - if (multiShape) { - extent = new Extent(builder.top, builder.bottom, builder.negLeft, builder.negRight, builder.posLeft, builder.posRight); - } - out.writeOptionalWriteable(extent); - out.writeVInt(builder.shapeWriters.size()); - if (multiShape) { - for (ShapeTreeWriter writer : builder.shapeWriters) { - try(BytesStreamOutput bytesStream = new BytesStreamOutput()) { - bytesStream.writeEnum(writer.getShapeType()); - writer.writeTo(bytesStream); - BytesReference bytes = bytesStream.bytes(); - out.writeVInt(bytes.length()); - bytes.writeTo(out); - } - } - } else { - out.writeEnum(builder.shapeWriters.get(0).getShapeType()); - builder.shapeWriters.get(0).writeTo(out); - } - } - - class GeometryTreeBuilder implements GeometryVisitor { - - private List shapeWriters; - private final CoordinateEncoder coordinateEncoder; - // integers are used to represent int-encoded lat/lon values - int top = Integer.MIN_VALUE; - int bottom = Integer.MAX_VALUE; - int negLeft = Integer.MAX_VALUE; - int negRight = Integer.MIN_VALUE; - int posLeft = Integer.MAX_VALUE; - int posRight = Integer.MIN_VALUE; - - GeometryTreeBuilder(CoordinateEncoder coordinateEncoder) { - this.coordinateEncoder = coordinateEncoder; - this.shapeWriters = new ArrayList<>(); - } - - private void addWriter(ShapeTreeWriter writer) { - Extent extent = writer.getExtent(); - top = Math.max(top, extent.top); - bottom = Math.min(bottom, extent.bottom); - negLeft = Math.min(negLeft, extent.negLeft); - negRight = Math.max(negRight, extent.negRight); - posLeft = Math.min(posLeft, extent.posLeft); - posRight = Math.max(posRight, extent.posRight); - shapeWriters.add(writer); - centroidCalculator.addFrom(writer.getCentroidCalculator()); - } - - @Override - public Void visit(GeometryCollection collection) { - addWriter(new GeometryTreeWriter(collection, coordinateEncoder)); - return null; - } - - @Override - public Void visit(Line line) { - addWriter(new EdgeTreeWriter(line.getLons(), line.getLats(), coordinateEncoder, false)); - return null; - } - - @Override - public Void visit(MultiLine multiLine) { - int size = multiLine.size(); - List x = new ArrayList<>(size); - List y = new ArrayList<>(size); - for (Line line : multiLine) { - x.add(line.getLons()); - y.add(line.getLats()); - } - addWriter(new EdgeTreeWriter(x, y, coordinateEncoder, false)); - return null; - } - - @Override - public Void visit(Polygon polygon) { - LinearRing outerShell = polygon.getPolygon(); - int numHoles = polygon.getNumberOfHoles(); - List x = new ArrayList<>(numHoles); - List y = new ArrayList<>(numHoles); - for (int i = 0; i < numHoles; i++) { - LinearRing innerRing = polygon.getHole(i); - x.add(innerRing.getLons()); - y.add(innerRing.getLats()); - } - addWriter(new PolygonTreeWriter(outerShell.getLons(), outerShell.getLats(), x, y, coordinateEncoder)); - return null; - } - - @Override - public Void visit(MultiPolygon multiPolygon) { - for (Polygon polygon : multiPolygon) { - visit(polygon); - } - return null; - } - - @Override - public Void visit(Rectangle r) { - double[] lats = new double[] { r.getMinLat(), r.getMinLat(), r.getMaxLat(), r.getMaxLat(), r.getMinLat() }; - double[] lons = new double[] { r.getMinLon(), r.getMaxLon(), r.getMaxLon(), r.getMinLon(), r.getMinLon() }; - addWriter(new PolygonTreeWriter(lons, lats, Collections.emptyList(), Collections.emptyList(), coordinateEncoder)); - return null; - } - - @Override - public Void visit(Point point) { - Point2DWriter writer = new Point2DWriter(point.getLon(), point.getLat(), coordinateEncoder); - addWriter(writer); - return null; - } - - @Override - public Void visit(MultiPoint multiPoint) { - double[] x = new double[multiPoint.size()]; - double[] y = new double[x.length]; - for (int i = 0; i < multiPoint.size(); i++) { - x[i] = multiPoint.get(i).getLon(); - y[i] = multiPoint.get(i).getLat(); - } - Point2DWriter writer = new Point2DWriter(x, y, coordinateEncoder); - addWriter(writer); - return null; - } - - @Override - public Void visit(LinearRing ring) { - throw new IllegalArgumentException("invalid shape type found [LinearRing]"); - } - - @Override - public Void visit(Circle circle) { - throw new IllegalArgumentException("invalid shape type found [Circle]"); - } - } -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java deleted file mode 100644 index 707a85513b933..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DReader.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; - -import java.io.IOException; -import java.util.ArrayDeque; -import java.util.Deque; - -/** - * This {@link ShapeTreeReader} understands how to parse points - * serialized with the {@link Point2DWriter} - */ -class Point2DReader implements ShapeTreeReader { - private final ByteBufferStreamInput input; - private final int size; - private final int startPosition; - - Point2DReader(ByteBufferStreamInput input) throws IOException { - this.input = input; - this.size = input.readVInt(); - this.startPosition = input.position(); - } - - @Override - public Extent getExtent() throws IOException { - if (size == 1) { - int x = readX(0); - int y = readY(0); - return Extent.fromPoint(x, y); - } else { - input.position(startPosition); - return new Extent(input); - } - } - - @Override - public double getCentroidX() { - throw new UnsupportedOperationException(); - } - - @Override - public double getCentroidY() { - throw new UnsupportedOperationException(); - } - - - @Override - public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { - Deque stack = new ArrayDeque<>(); - stack.push(0); - stack.push(size - 1); - stack.push(0); - while (stack.isEmpty() == false) { - int axis = stack.pop(); - int right = stack.pop(); - int left = stack.pop(); - - if (right - left <= Point2DWriter.LEAF_SIZE) { - for (int i = left; i <= right; i++) { - // TODO serialize to re-usable array instead of serializing in each step - int x = readX(i); - int y = readY(i); - if (x >= minX && x <= maxX && y >= minY && y <= maxY) { - return GeoRelation.QUERY_CROSSES; - } - } - continue; - } - - int middle = (right + left) >> 1; - int x = readX(middle); - int y = readY(middle); - if (x >= minX && x <= maxX && y >= minY && y <= maxY) { - return GeoRelation.QUERY_CROSSES; - } - if ((axis == 0 && minX <= x) || (axis == 1 && minY <= y)) { - stack.push(left); - stack.push(middle - 1); - stack.push(1 - axis); - } - if ((axis == 0 && maxX >= x) || (axis == 1 && maxY >= y)) { - stack.push(middle + 1); - stack.push(right); - stack.push(1 - axis); - } - } - - return GeoRelation.QUERY_DISJOINT; - } - - private int readX(int pointIdx) throws IOException { - int extentOffset = size == 1 ? 0 : Extent.WRITEABLE_SIZE_IN_BYTES; - input.position(startPosition + extentOffset + 2 * pointIdx * Integer.BYTES); - return input.readInt(); - } - - private int readY(int pointIdx) throws IOException { - int extentOffset = size == 1 ? 0 : Extent.WRITEABLE_SIZE_IN_BYTES; - input.position(startPosition + extentOffset + (2 * pointIdx + 1) * Integer.BYTES); - return input.readInt(); - } -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java b/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java deleted file mode 100644 index bd2a250925b78..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/Point2DWriter.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.geometry.ShapeType; - -import java.io.IOException; - -/** - * points KD-Tree (2D) writer for use in doc-values. - * - * This work is influenced by https://github.com/mourner/kdbush (ISC licensed). - */ -public class Point2DWriter extends ShapeTreeWriter { - - private static final int K = 2; - private final Extent extent; - private final double[] coords; - // size of a leaf node where searches are done sequentially. - static final int LEAF_SIZE = 64; - private final CoordinateEncoder coordinateEncoder; - private final CentroidCalculator centroidCalculator; - - Point2DWriter(double[] x, double[] y, CoordinateEncoder coordinateEncoder) { - assert x.length == y.length; - this.coordinateEncoder = coordinateEncoder; - this.centroidCalculator = new CentroidCalculator(); - double top = Double.NEGATIVE_INFINITY; - double bottom = Double.POSITIVE_INFINITY; - double negLeft = Double.POSITIVE_INFINITY; - double negRight = Double.NEGATIVE_INFINITY; - double posLeft = Double.POSITIVE_INFINITY; - double posRight = Double.NEGATIVE_INFINITY; - coords = new double[x.length * K]; - - for (int i = 0; i < x.length; i++) { - double xi = x[i]; - double yi = y[i]; - top = Math.max(top, yi); - bottom = Math.min(bottom, yi); - if (xi >= 0 && xi < posLeft) { - posLeft = xi; - } - if (xi >= 0 && xi > posRight) { - posRight = xi; - } - if (xi < 0 && xi < negLeft) { - negLeft = xi; - } - if (xi < 0 && xi > negRight) { - negRight = xi; - } - coords[2 * i] = xi; - coords[2 * i + 1] = yi; - - centroidCalculator.addCoordinate(xi, yi); - } - sort(0, x.length - 1, 0); - this.extent = new Extent(coordinateEncoder.encodeY(top), coordinateEncoder.encodeY(bottom), coordinateEncoder.encodeX(negLeft), - coordinateEncoder.encodeX(negRight), coordinateEncoder.encodeX(posLeft), coordinateEncoder.encodeX(posRight)); - } - - Point2DWriter(double x, double y, CoordinateEncoder coordinateEncoder) { - this.coordinateEncoder = coordinateEncoder; - coords = new double[] {x, y}; - this.extent = Extent.fromPoint(coordinateEncoder.encodeX(x), coordinateEncoder.encodeY(y)); - this.centroidCalculator = new CentroidCalculator(); - centroidCalculator.addCoordinate(x, y); - } - - @Override - public Extent getExtent() { - return extent; - } - - @Override - public ShapeType getShapeType() { - return ShapeType.MULTIPOINT; - } - - @Override - public CentroidCalculator getCentroidCalculator() { - return centroidCalculator; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - int numPoints = coords.length >> 1; - out.writeVInt(numPoints); - if (numPoints > 1) { - extent.writeTo(out); - } - for (int i = 0; i < coords.length; i++) { - double coord = coords[i]; - int encodedCoord = i % 2 == 0 ? coordinateEncoder.encodeX(coord) : coordinateEncoder.encodeY(coord); - out.writeInt(encodedCoord); - } - } - - private void sort(int left, int right, int depth) { - // since the reader will search through points within a leaf, - // there is no improved performance by sorting these points. - if (right - left <= LEAF_SIZE) { - return; - } - - int middle = (left + right) >> 1; - - select(left, right, middle, depth); - - sort(left, middle - 1, depth + 1); - sort(middle + 1, right, depth + 1); - } - - /** - * A slightly-modified Floyd-Rivest selection algorithm, - * https://en.wikipedia.org/wiki/Floyd%E2%80%93Rivest_algorithm - * - * @param left the index of the left point - * @param right the index of the right point - * @param k the pivot index - * @param depth the depth in the kd-tree - */ - private void select(int left, int right, int k, int depth) { - int axis = depth % K; - while (right > left) { - if (right - left > 600) { - double n = right - left + 1; - int i = k - left + 1; - double z = Math.log(n); - double s = 0.5 * Math.exp(2 * z / 3); - double sd = 0.5 * Math.sqrt(z * s * (n - s) / n) * ((i - n / 2) < 0 ? -1 : 1); - int newLeft = Math.max(left, (int) Math.floor(k - i * s / n + sd)); - int newRight = Math.min(right, (int) Math.floor(k + (n - i) * s / n + sd)); - select(newLeft, newRight, k, depth); - } - double t = coords[2 * k + axis]; - int i = left; - int j = right; - - swapPoint(left, k); - if (coords[2 * right + axis] > t) { - swapPoint(left, right); - } - - while (i < j) { - swapPoint(i, j); - i++; - j--; - while (coords[2 * i + axis] < t) { - i++; - } - while (coords[2 * j + axis] > t) { - j--; - } - } - - if (coords[2 * left + axis] == t) { - swapPoint(left, j); - } else { - j++; - swapPoint(j, right); - } - - if (j <= k) { - left = j + 1; - } - if (k <= j) { - right = j - 1; - } - } - } - - private void swapPoint(int i, int j) { - swap( 2 * i, 2 * j); - swap(2 * i + 1, 2 * j + 1); - } - - private void swap(int i, int j) { - double tmp = coords[i]; - coords[i] = coords[j]; - coords[j] = tmp; - } -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java deleted file mode 100644 index e330c0f8e1d4d..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeReader.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; - -import java.io.IOException; - -/** - * This {@link ShapeTreeReader} understands how to parse polygons - * serialized with the {@link PolygonTreeWriter} - */ -public class PolygonTreeReader implements ShapeTreeReader { - private final EdgeTreeReader outerShell; - private final EdgeTreeReader holes; - - public PolygonTreeReader(ByteBufferStreamInput input) throws IOException { - int outerShellSize = input.readVInt(); - int outerShellPosition = input.position(); - this.outerShell = new EdgeTreeReader(input, true); - input.position(outerShellPosition + outerShellSize); - boolean hasHoles = input.readBoolean(); - if (hasHoles) { - this.holes = new EdgeTreeReader(input, true); - } else { - this.holes = null; - } - } - - public Extent getExtent() throws IOException { - return outerShell.getExtent(); - } - - @Override - public double getCentroidX() { - throw new UnsupportedOperationException(); - } - - @Override - public double getCentroidY() { - throw new UnsupportedOperationException(); - } - - /** - * Returns true if the rectangle query and the edge tree's shape overlap - */ - @Override - public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { - if (holes != null) { - GeoRelation relation = holes.relate(minX, minY, maxX, maxY); - if (GeoRelation.QUERY_CROSSES == relation) { - return GeoRelation.QUERY_CROSSES; - } - if (GeoRelation.QUERY_INSIDE == relation) { - return GeoRelation.QUERY_DISJOINT; - } - } - return outerShell.relate(minX, minY, maxX, maxY); - } -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java deleted file mode 100644 index 1a48821afcafc..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/PolygonTreeWriter.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.ShapeType; - -import java.io.IOException; -import java.util.List; - -/** - * {@link Polygon} and {@link Rectangle} Shape Tree Writer for use in doc-values - */ -public class PolygonTreeWriter extends ShapeTreeWriter { - - private final EdgeTreeWriter outerShell; - private final EdgeTreeWriter holes; - - public PolygonTreeWriter(double[] x, double[] y, List holesX, List holesY, CoordinateEncoder coordinateEncoder) { - outerShell = new EdgeTreeWriter(x, y, coordinateEncoder, true); - holes = holesX.isEmpty() ? null : new EdgeTreeWriter(holesX, holesY, coordinateEncoder, true); - } - - public Extent getExtent() { - return outerShell.getExtent(); - } - - public ShapeType getShapeType() { - return ShapeType.POLYGON; - } - - @Override - public CentroidCalculator getCentroidCalculator() { - return outerShell.getCentroidCalculator(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - // calculate size of outerShell's tree to make it easy to jump to the holes tree quickly when querying - BytesStreamOutput scratchBuffer = new BytesStreamOutput(); - outerShell.writeTo(scratchBuffer); - int outerShellSize = scratchBuffer.size(); - out.writeVInt(outerShellSize); - long startPosition = out.position(); - outerShell.writeTo(out); - assert out.position() == outerShellSize + startPosition; - out.writeOptionalWriteable(holes); - } -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java deleted file mode 100644 index 743ae712b4012..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeReader.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.geometry.Geometry; - -import java.io.IOException; - -/** - * Shape Reader to read different {@link Geometry} doc-values - */ -public interface ShapeTreeReader { - - Extent getExtent() throws IOException; - GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException; - double getCentroidX() throws IOException; - double getCentroidY() throws IOException; -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index 8afae43b8da9c..b44bc3cb687ef 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -33,7 +33,7 @@ * This class supports checking bounding box * relations against the serialized triangle tree. */ -public class TriangleTreeReader implements ShapeTreeReader { +public class TriangleTreeReader { private final int extentOffset = 8; private ByteBufferStreamInput input; @@ -76,7 +76,6 @@ public Extent getExtent() throws IOException { /** * returns the X coordinate of the centroid. */ - @Override public double getCentroidX() throws IOException { input.position(0); return coordinateEncoder.decodeX(input.readInt()); @@ -85,7 +84,6 @@ public double getCentroidX() throws IOException { /** * returns the Y coordinate of the centroid. */ - @Override public double getCentroidY() throws IOException { input.position(4); return coordinateEncoder.decodeY(input.readInt()); @@ -95,7 +93,6 @@ public double getCentroidY() throws IOException { * Compute the relation with the provided bounding box. If the result is CELL_INSIDE_QUERY * then the bounding box is within the shape. */ - @Override public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { Extent extent = getExtent(); int thisMaxX = extent.maxX(); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 78829cca7dbb9..893d020ebc657 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -24,9 +24,8 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoRelation; import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; -import org.elasticsearch.common.geo.GeometryTreeReader; -import org.elasticsearch.common.geo.GeometryTreeWriter; -import org.elasticsearch.common.geo.ShapeTreeReader; +import org.elasticsearch.common.geo.TriangleTreeReader; +import org.elasticsearch.common.geo.TriangleTreeWriter; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; @@ -127,9 +126,9 @@ public String toString() { public static class GeoShapeValue implements GeoValue { private static final WellKnownText MISSING_GEOMETRY_PARSER = new WellKnownText(true, new GeographyValidator(true)); - private final ShapeTreeReader reader; + private final TriangleTreeReader reader; - public GeoShapeValue(ShapeTreeReader reader) { + public GeoShapeValue(TriangleTreeReader reader) { this.reader = reader; } @@ -182,10 +181,10 @@ public double lon() { public static GeoShapeValue missing(String missing) { try { Geometry geometry = MISSING_GEOMETRY_PARSER.fromWKT(missing); - GeometryTreeWriter writer = new GeometryTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); + TriangleTreeWriter writer = new TriangleTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); - GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + TriangleTreeReader reader = new TriangleTreeReader(GeoShapeCoordinateEncoder.INSTANCE); reader.reset(output.bytes().toBytesRef()); return new GeoShapeValue(reader); } catch (IOException | ParseException e) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java index 80837f098caaa..51cc6cdd53765 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LatLonShapeDVAtomicFieldData.java @@ -24,7 +24,7 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; -import org.elasticsearch.common.geo.GeometryTreeReader; +import org.elasticsearch.common.geo.TriangleTreeReader; import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -62,7 +62,7 @@ public void close() { public MultiGeoValues getGeoValues() { try { final BinaryDocValues binaryValues = DocValues.getBinary(reader, fieldName); - final GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + final TriangleTreeReader reader = new TriangleTreeReader(GeoShapeCoordinateEncoder.INSTANCE); final MultiGeoValues.GeoShapeValue geoShapeValue = new MultiGeoValues.GeoShapeValue(reader); return new MultiGeoValues() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java index 619c7272b4fa7..9702fcb2403e3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java @@ -21,7 +21,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; -import org.elasticsearch.common.geo.GeometryTreeWriter; +import org.elasticsearch.common.geo.TriangleTreeWriter; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; @@ -53,7 +53,7 @@ public BytesRef binaryValue() { } else { geometry = geometries.get(0); } - final GeometryTreeWriter writer = new GeometryTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); + final TriangleTreeWriter writer = new TriangleTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); return output.bytes().toBytesRef(); diff --git a/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java b/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java deleted file mode 100644 index 1162782c329d6..0000000000000 --- a/server/src/test/java/org/elasticsearch/common/geo/AbstractTreeTestCase.java +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.GeometryCollection; -import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; -import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.index.mapper.GeoShapeIndexer; -import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.geo.RandomShapeGenerator; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.Function; - -import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; -import static org.elasticsearch.geo.GeometryTestUtils.fold; -import static org.elasticsearch.geo.GeometryTestUtils.randomPoint; -import static org.hamcrest.Matchers.equalTo; - -public abstract class AbstractTreeTestCase extends ESTestCase { - - public void testRectangleShape() throws IOException { - for (int i = 0; i < 1000; i++) { - int minX = randomIntBetween(-80, 70); - int maxX = randomIntBetween(minX + 10, 80); - int minY = randomIntBetween(-80, 70); - int maxY = randomIntBetween(minY + 10, 80); - double[] x = new double[]{minX, maxX, maxX, minX, minX}; - double[] y = new double[]{minY, minY, maxY, maxY, minY}; - Geometry rectangle = randomBoolean() ? - new Polygon(new LinearRing(x, y), Collections.emptyList()) : new Rectangle(minX, maxX, maxY, minY); - ShapeTreeReader reader = geometryTreeReader(rectangle, TestCoordinateEncoder.INSTANCE); - - assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); - // encoder loses precision when casting to integer, so centroid is calculated using integer division here - assertThat(reader.getCentroidX(), equalTo((double) ((minX + maxX) / 2))); - assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY) / 2))); - - // box-query touches bottom-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), - minY - randomIntBetween(1, 180), minX, minY)); - // box-query touches bottom-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), - maxX + randomIntBetween(1, 180), minY)); - // box-query touches top-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), - maxY + randomIntBetween(1, 180))); - // box-query touches top-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, - maxY + randomIntBetween(1, 180))); - // box-query fully-enclosed inside rectangle - assertRelation(GeoRelation.QUERY_INSIDE,reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); - // box-query fully-contains poly - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), - minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); - // box-query half-in-half-out-right - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); - // box-query half-in-half-out-left - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, - (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); - // box-query half-in-half-out-top - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - maxX + randomIntBetween(1, 1000), maxY + randomIntBetween(1, 1000))); - // box-query half-in-half-out-bottom - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); - - // box-query outside to the right - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, - maxX + randomIntBetween(1001, 2000), maxY)); - // box-query outside to the left - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, - minX - randomIntBetween(1, 1000), maxY)); - // box-query outside to the top - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, - maxY + randomIntBetween(1001, 2000))); - // box-query outside to the bottom - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, - minY - randomIntBetween(1, 1000))); - } - } - - public void testSimplePolygon() throws IOException { - for (int iter = 0; iter < 1000; iter++) { - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "name"); - ShapeBuilder builder = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON); - Polygon geo = (Polygon) builder.buildGeometry(); - Geometry geometry = indexer.prepareForIndexing(geo); - Polygon testPolygon; - if (geometry instanceof Polygon) { - testPolygon = (Polygon) geometry; - } else if (geometry instanceof MultiPolygon) { - testPolygon = ((MultiPolygon) geometry).get(0); - } else { - throw new IllegalStateException("not a polygon"); - } - builder = LegacyGeoShapeQueryProcessor.geometryToShapeBuilder(testPolygon); - org.locationtech.spatial4j.shape.Rectangle box = builder.buildS4J().getBoundingBox(); - int minXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMinX()); - int minYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMinY()); - int maxXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMaxX()); - int maxYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMaxY()); - - double[] x = testPolygon.getPolygon().getLons(); - double[] y = testPolygon.getPolygon().getLats(); - - EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE, true); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); - Extent actualExtent = reader.getExtent(); - assertThat(actualExtent.minX(), equalTo(minXBox)); - assertThat(actualExtent.maxX(), equalTo(maxXBox)); - assertThat(actualExtent.minY(), equalTo(minYBox)); - assertThat(actualExtent.maxY(), equalTo(maxYBox)); - // polygon fully contained within box - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox)); - // relate - if (maxYBox - 1 >= minYBox) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1)); - } - if (maxXBox -1 >= minXBox) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox)); - } - // does not cross - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10)); - } - } - - public void testPacManPolygon() throws Exception { - // pacman - double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - - // test cell crossing poly - ShapeTreeReader reader = geometryTreeReader(new Polygon(new LinearRing(py, px), Collections.emptyList()), - TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(-5, -6, 2, -2)); - } - - // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon - public void testPolygonWithHole() throws Exception { - Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), - Collections.singletonList(new LinearRing(new double[]{-10, 10, 10, -10, -10}, new double[]{-10, -10, 10, 10, -10}))); - - ShapeTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); - - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(6, -6, 6, -6)); // in the hole - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(25, -25, 25, -25)); // on the mainland - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(51, 51, 52, 52)); // outside of mainland - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-60, -60, 60, 60)); // enclosing us completely - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(49, 49, 51, 51)); // overlapping the mainland - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(9, 9, 11, 11)); // overlapping the hole - } - - public void testCombPolygon() throws Exception { - double[] px = {0, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 0, 0}; - double[] py = {0, 0, 20, 20, 0, 0, 20, 20, 0, 0, 30, 30, 0}; - - double[] hx = {21, 21, 29, 29, 21}; - double[] hy = {1, 20, 20, 1, 1}; - - Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); - ShapeTreeReader reader = geometryTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); - // test cell crossing poly - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(5, 10, 5, 10)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(15, 10, 15, 10)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(25, 10, 25, 10)); - } - - public void testPacManClosedLineString() throws Exception { - // pacman - double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - - // test cell crossing poly - ShapeTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); - } - - public void testPacManLineString() throws Exception { - // pacman - double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; - - // test cell crossing poly - ShapeTreeReader reader = geometryTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); - } - - public void testPacManPoints() throws Exception { - // pacman - List points = Arrays.asList( - new Point(0, 0), - new Point(5, 10), - new Point(9, 10), - new Point(10, 0), - new Point(9, -8), - new Point(0, -10), - new Point(-9, -8), - new Point(-10, 0), - new Point(-9, 10), - new Point(-5, 10) - ); - - - // candidate intersects cell - int xMin = 0; - int xMax = 11; - int yMin = -10; - int yMax = 9; - - // test cell crossing poly - ShapeTreeReader reader = geometryTreeReader(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(xMin, yMin, xMax, yMax)); - } - - public void testRandomMultiLineIntersections() throws IOException { - double extentSize = randomDoubleBetween(0.01, 10, true); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - MultiLine geometry = GeometryTestUtils.randomMultiLine(false); - geometry = (MultiLine) indexer.prepareForIndexing(geometry); - - ShapeTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); - Extent readerExtent = reader.getExtent(); - - for (Line line : geometry) { - // extent that intersects edges - assertRelation(GeoRelation.QUERY_CROSSES, reader, bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize)); - - // extent that fully encloses a line in the MultiLine - Extent lineExtent = geometryTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); - assertRelation(GeoRelation.QUERY_CROSSES, reader, lineExtent); - - if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE - && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, - lineExtent.maxX() + 1, lineExtent.maxY() + 1)); - } - } - - // extent that fully encloses the MultiLine - assertRelation(GeoRelation.QUERY_CROSSES, reader, reader.getExtent()); - if (readerExtent.minX() != Integer.MIN_VALUE && readerExtent.maxX() != Integer.MAX_VALUE - && readerExtent.minY() != Integer.MIN_VALUE && readerExtent.maxY() != Integer.MAX_VALUE) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, - readerExtent.maxX() + 1, readerExtent.maxY() + 1)); - } - - } - - public void testRandomGeometryIntersection() throws IOException { - int testPointCount = randomIntBetween(100, 200); - Point[] testPoints = new Point[testPointCount]; - double extentSize = randomDoubleBetween(1, 10, true); - boolean[] intersects = new boolean[testPointCount]; - for (int i = 0; i < testPoints.length; i++) { - testPoints[i] = randomPoint(false); - } - - Geometry geometry = randomGeometryTreeGeometry(); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - Geometry preparedGeometry = indexer.prepareForIndexing(geometry); - - for (int i = 0; i < testPointCount; i++) { - int cur = i; - intersects[cur] = fold(preparedGeometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); - } - - for (int i = 0; i < testPointCount; i++) { - assertEquals(intersects[i], intersects(preparedGeometry, testPoints[i], extentSize)); - } - } - - private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) { - int xMin = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.max(x - extentSize, -180.0)); - int xMax = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.min(x + extentSize, 180.0)); - int yMin = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.max(y - extentSize, -90)); - int yMax = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.min(y + extentSize, 90)); - return Extent.fromPoints(xMin, yMin, xMax, yMax); - } - - private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { - - Extent bufferBounds = bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize); - GeoRelation relation = geometryTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) - .relate(bufferBounds.minX(), bufferBounds.minY(), bufferBounds.maxX(), bufferBounds.maxY()); - return relation == GeoRelation.QUERY_CROSSES || relation == GeoRelation.QUERY_INSIDE; - } - - private static Geometry randomGeometryTreeGeometry() { - return randomGeometryTreeGeometry(0); - } - - private static Geometry randomGeometryTreeGeometry(int level) { - @SuppressWarnings("unchecked") Function geometry = ESTestCase.randomFrom( - GeometryTestUtils::randomLine, - GeometryTestUtils::randomPoint, - GeometryTestUtils::randomPolygon, - GeometryTestUtils::randomMultiLine, - GeometryTestUtils::randomMultiPoint, - level < 3 ? (b) -> randomGeometryTreeCollection(level + 1) : GeometryTestUtils::randomPoint // don't build too deep - ); - return geometry.apply(false); - } - - private static Geometry randomGeometryTreeCollection(int level) { - int size = ESTestCase.randomIntBetween(1, 10); - List shapes = new ArrayList<>(); - for (int i = 0; i < size; i++) { - shapes.add(randomGeometryTreeGeometry(level)); - } - return new GeometryCollection<>(shapes); - } - - protected abstract ShapeTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException; -} diff --git a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java deleted file mode 100644 index 5901fa0228d25..0000000000000 --- a/server/src/test/java/org/elasticsearch/common/geo/EdgeTreeTests.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.ShapeType; -import org.elasticsearch.index.mapper.GeoShapeIndexer; -import org.elasticsearch.index.query.LegacyGeoShapeQueryProcessor; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.geo.RandomShapeGenerator; -import org.locationtech.spatial4j.shape.Rectangle; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; - -import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; -import static org.hamcrest.Matchers.equalTo; - -public class EdgeTreeTests extends ESTestCase { - - public void testRectangleShape() throws IOException { - for (int i = 0; i < 1000; i++) { - int minX = randomIntBetween(-180, 170); - int maxX = randomIntBetween(minX + 10, 180); - int minY = randomIntBetween(-180, 170); - int maxY = randomIntBetween(minY + 10, 180); - double[] x = new double[]{minX, maxX, maxX, minX, minX}; - double[] y = new double[]{minY, minY, maxY, maxY, minY}; - EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE, true); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - EdgeTreeReader reader = new EdgeTreeReader( - new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); - - assertThat(writer.getCentroidCalculator().getX(), equalTo((minX + maxX)/2.0)); - assertThat(writer.getCentroidCalculator().getY(), equalTo((minY + maxY)/2.0)); - - - // box-query touches bottom-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), - minY - randomIntBetween(1, 180), minX, minY)); - // box-query touches bottom-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), - maxX + randomIntBetween(1, 180), minY)); - // box-query touches top-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), - maxY + randomIntBetween(1, 180))); - // box-query touches top-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, - maxY + randomIntBetween(1, 180))); - // box-query fully-enclosed inside rectangle - assertRelation(GeoRelation.QUERY_INSIDE,reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); - // box-query fully-contains poly - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), - minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); - // box-query half-in-half-out-right - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); - // box-query half-in-half-out-left - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, - (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); - // box-query half-in-half-out-top - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - maxX + randomIntBetween(1, 1000), maxY + randomIntBetween(1, 1000))); - // box-query half-in-half-out-bottom - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); - - // box-query outside to the right - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, - maxX + randomIntBetween(1001, 2000), maxY)); - // box-query outside to the left - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, - minX - randomIntBetween(1, 1000), maxY)); - // box-query outside to the top - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, - maxY + randomIntBetween(1001, 2000))); - // box-query outside to the bottom - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, - minY - randomIntBetween(1, 1000))); - } - } - - public void testSimplePolygon() throws IOException { - for (int iter = 0; iter < 1000; iter++) { - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "name"); - ShapeBuilder builder = RandomShapeGenerator.createShape(random(), RandomShapeGenerator.ShapeType.POLYGON); - Polygon geo = (Polygon) builder.buildGeometry(); - Geometry geometry = indexer.prepareForIndexing(geo); - Polygon testPolygon; - if (geometry instanceof Polygon) { - testPolygon = (Polygon) geometry; - } else if (geometry instanceof MultiPolygon) { - testPolygon = ((MultiPolygon) geometry).get(0); - } else { - throw new IllegalStateException("not a polygon"); - } - builder = LegacyGeoShapeQueryProcessor.geometryToShapeBuilder(testPolygon); - Rectangle box = builder.buildS4J().getBoundingBox(); - int minXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMinX()); - int minYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMinY()); - int maxXBox = TestCoordinateEncoder.INSTANCE.encodeX(box.getMaxX()); - int maxYBox = TestCoordinateEncoder.INSTANCE.encodeY(box.getMaxY()); - - double[] x = testPolygon.getPolygon().getLons(); - double[] y = testPolygon.getPolygon().getLats(); - - EdgeTreeWriter writer = new EdgeTreeWriter(x, y, TestCoordinateEncoder.INSTANCE, true); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); - Extent actualExtent = reader.getExtent(); - assertThat(actualExtent.minX(), equalTo(minXBox)); - assertThat(actualExtent.maxX(), equalTo(maxXBox)); - assertThat(actualExtent.minY(), equalTo(minYBox)); - assertThat(actualExtent.maxY(), equalTo(maxYBox)); - // polygon fully contained within box - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox)); - // relate - if (maxYBox - 1 >= minYBox) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox, maxYBox - 1)); - } - if (maxXBox - 1 >= minXBox) { - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minXBox, minYBox, maxXBox - 1, maxYBox)); - } - // does not cross - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxXBox + 1, maxYBox + 1, maxXBox + 10, maxYBox + 10)); - } - } - - public void testPacMan() throws Exception { - // pacman - double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; - - // candidate relate cell - int xMin = 2;//-5; - int xMax = 11;//0.000001; - int yMin = -1;//0; - int yMax = 1;//5; - - // test cell crossing poly - EdgeTreeWriter writer = new EdgeTreeWriter(px, py, TestCoordinateEncoder.INSTANCE, true); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - EdgeTreeReader reader = new EdgeTreeReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes)), true); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(xMin, yMin, xMax, yMax)); - } - - public void testGetShapeType() { - double[] pointCoord = new double[] { 0 }; - assertThat(new EdgeTreeWriter(pointCoord, pointCoord, TestCoordinateEncoder.INSTANCE, false).getShapeType(), - equalTo(ShapeType.LINESTRING)); - assertThat(new EdgeTreeWriter(List.of(pointCoord, pointCoord), List.of(pointCoord, pointCoord), - TestCoordinateEncoder.INSTANCE, false).getShapeType(), - equalTo(ShapeType.MULTILINESTRING)); - } -} diff --git a/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java b/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java deleted file mode 100644 index 675620ecf8db6..0000000000000 --- a/server/src/test/java/org/elasticsearch/common/geo/EncodingComparisonTests.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.apache.lucene.geo.GeoTestUtil; -import org.apache.lucene.geo.Tessellator; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TestUtil; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.utils.Geohash; -import org.elasticsearch.test.ESTestCase; -import org.locationtech.spatial4j.io.GeohashUtils; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - - -// This test class is just for comparing the size of different encodings. -@LuceneTestCase.AwaitsFix(bugUrl = "this is just for reference") -public class EncodingComparisonTests extends ESTestCase { - - public void testRandomRectangle() throws Exception { - for (int i =0; i < 100; i++) { - org.apache.lucene.geo.Rectangle random = GeoTestUtil.nextBox(); - compareWriters(new Rectangle(random.minLon, random.maxLon, random.maxLat, random.minLat), - new GeoShapeCoordinateEncoder()); - } - } - - public void testRandomPolygon() throws Exception { - for (int i =0; i < 100; i++) { - org.apache.lucene.geo.Polygon random = GeoTestUtil.nextPolygon(); - while (true) { - try { - Tessellator.tessellate(random); - break; - } catch (Exception e) { - random = GeoTestUtil.nextPolygon(); - } - } - compareWriters(new Polygon(new LinearRing(random.getPolyLons(), random.getPolyLats()), - Collections.emptyList()), new GeoShapeCoordinateEncoder()); - } - } - - public void testRandomMultiPolygon() throws Exception { - for (int i =0; i < 100; i++) { - int n = TestUtil.nextInt(random(), 2, 10); - List polygons = new ArrayList<>(n); - for (int j =0; j < n; j++) { - org.apache.lucene.geo.Polygon random = GeoTestUtil.nextPolygon(); - while (true) { - try { - Tessellator.tessellate(random); - break; - } catch (Exception e) { - random = GeoTestUtil.nextPolygon(); - } - } - polygons.add(new Polygon(new LinearRing(random.getPolyLons(), random.getPolyLats()), - Collections.emptyList())); - } - MultiPolygon multiPolygon = new MultiPolygon(polygons); - compareWriters(multiPolygon, new GeoShapeCoordinateEncoder()); - } - } - - public void testRandomLine() throws Exception { - for (int i =0; i < 100; i++) { - double[] px = new double[2]; - double[] py = new double[2]; - for (int j =0; j < 2; j++) { - px[j] = GeoTestUtil.nextLongitude(); - px[j] = GeoTestUtil.nextLatitude(); - } - compareWriters(new Line(px, py), new GeoShapeCoordinateEncoder()); - } - } - - public void testRandomLines() throws Exception { - for (int i =0; i < 100; i++) { - int numPoints = TestUtil.nextInt(random(), 2, 1000); - double[] px = new double[numPoints]; - double[] py = new double[numPoints]; - for (int j =0; j < numPoints; j++) { - px[j] = GeoTestUtil.nextLongitude(); - px[j] = GeoTestUtil.nextLatitude(); - } - compareWriters(new Line(px, py), new GeoShapeCoordinateEncoder()); - } - } - - public void testRandomPoint() throws Exception { - for (int i =0; i < 100; i++) { - compareWriters(new Point(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude()), - new GeoShapeCoordinateEncoder()); - } - } - - public void testRandomPoints() throws Exception { - for (int i =0; i < 100; i++) { - int numPoints = TestUtil.nextInt(random(), 2, 1000); - List points = new ArrayList<>(numPoints); - - for (int j =0; j < numPoints; j++) { - points.add(new Point(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude())); - } - compareWriters(new MultiPoint(points), new GeoShapeCoordinateEncoder()); - } - } - - private void compareWriters(Geometry geometry, CoordinateEncoder encoder) throws Exception { - TriangleTreeWriter writer1 = new TriangleTreeWriter(geometry, encoder); - GeometryTreeWriter writer2 = new GeometryTreeWriter(geometry, encoder); - BytesStreamOutput output1 = new BytesStreamOutput(); - BytesStreamOutput output2 = new BytesStreamOutput(); - writer1.writeTo(output1); - writer2.writeTo(output2); - output1.close(); - output2.close(); - //String s1 = "Triangles: " + output1.bytes().length(); - //String s2 = "Edge tree: " + output2.bytes().length(); - // System.out.println(s1 + " / " + s2 + " / diff: " + (double) output1.bytes().length() / output2.bytes().length()); - - int maxPrecision = 5; - TriangleTreeReader reader1 = new TriangleTreeReader(encoder); - reader1.reset(output1.bytes().toBytesRef()); - //long t1 = System.nanoTime(); - int h1 = getHashesAtLevel(reader1, encoder, "", maxPrecision); - //long t2 = System.nanoTime(); - GeometryTreeReader reader2 = new GeometryTreeReader(encoder); - reader2.reset(output1.bytes().toBytesRef()); - //long t3 = System.nanoTime(); - int h2= getHashesAtLevel(reader2, encoder, "", maxPrecision); - //long t4 = System.nanoTime(); - assertEquals(h1, h2); - - //String s3 = "Triangles: " + h1; //(t2 - t1); - //String s4 = "Edge tree: " + h2; //(t4 - t3); - //System.out.println("Query: " + s3 + " / " + s4 + " / diff: " + (double) (t2 - t1) / (t4 - t3)); - } - - private int getHashesAtLevel(ShapeTreeReader reader, CoordinateEncoder encoder, String hash, int maxPrecision) throws IOException { - int hits = 0; - String[] hashes = GeohashUtils.getSubGeohashes(hash); - for (int i =0; i < hashes.length; i++) { - Rectangle r = Geohash.toBoundingBox(hashes[i]); - GeoRelation rel = reader.relate(encoder.encodeX(r.getMinLon()), - encoder.encodeY(r.getMinLat()), - encoder.encodeX(r.getMaxLon()), - encoder.encodeY(r.getMaxLat())); - if (rel == GeoRelation.QUERY_CROSSES) { - if (hashes[i].length() == maxPrecision) { - hits++; - } else { - hits += getHashesAtLevel(reader, encoder, hashes[i], maxPrecision); - } - } else if (rel == GeoRelation.QUERY_INSIDE) { - if (hashes[i].length() == maxPrecision) { - hits++; - } else { - hits += getHashesAtLevel(hashes[i], maxPrecision); - } - } - } - return hits; - } - - private int getHashesAtLevel(String hash, int maxPrecision) { - int hits = 0; - String[] hashes = GeohashUtils.getSubGeohashes(hash); - for (int i = 0; i < hashes.length; i++) { - if (hashes[i].length() == maxPrecision) { - hits++; - } else { - hits += getHashesAtLevel(hashes[i], maxPrecision); - } - } - return hits; - } -} diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java index 3b2da0e617c24..eb54ecba06ae4 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -34,17 +34,17 @@ public class GeoTestUtils { - public static void assertRelation(GeoRelation expectedRelation, ShapeTreeReader reader, Extent extent) throws IOException { + public static void assertRelation(GeoRelation expectedRelation, TriangleTreeReader reader, Extent extent) throws IOException { GeoRelation actualRelation = reader.relate(extent.minX(), extent.minY(), extent.maxX(), extent.maxY()); assertThat(actualRelation, equalTo(expectedRelation)); } - public static GeometryTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { - GeometryTreeWriter writer = new GeometryTreeWriter(geometry, encoder); + public static TriangleTreeReader triangleTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { + TriangleTreeWriter writer = new TriangleTreeWriter(geometry, encoder); BytesStreamOutput output = new BytesStreamOutput(); writer.writeTo(output); output.close(); - GeometryTreeReader reader = new GeometryTreeReader(encoder); + TriangleTreeReader reader = new TriangleTreeReader(encoder); reader.reset(output.bytes().toBytesRef()); return reader; } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java deleted file mode 100644 index 892571e1c4f3c..0000000000000 --- a/server/src/test/java/org/elasticsearch/common/geo/GeometryTreeTests.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geometry.Geometry; - -import java.io.IOException; - - -public class GeometryTreeTests extends AbstractTreeTestCase { - - protected ShapeTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { - GeometryTreeWriter writer = new GeometryTreeWriter(geometry, encoder); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(encoder); - reader.reset(output.bytes().toBytesRef()); - return reader; - } -} diff --git a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java b/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java deleted file mode 100644 index 01747a624d5f6..0000000000000 --- a/server/src/test/java/org/elasticsearch/common/geo/Point2DTests.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.test.ESTestCase; - -import java.io.IOException; -import java.nio.ByteBuffer; - -import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; -import static org.hamcrest.Matchers.equalTo; - -public class Point2DTests extends ESTestCase { - - public void testOnePoint() throws IOException { - int x = randomIntBetween(-90, 90); - int y = randomIntBetween(-90, 90); - Point2DWriter writer = new Point2DWriter(x, y, TestCoordinateEncoder.INSTANCE); - - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); - assertThat(reader.getExtent(), equalTo(Extent.fromPoint(x, y))); - assertThat(reader.getExtent(), equalTo(reader.getExtent())); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoint(x, y)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, - Extent.fromPoints(x, y, x + randomIntBetween(1, 10), y + randomIntBetween(1, 10))); - assertRelation(GeoRelation.QUERY_CROSSES, reader, - Extent.fromPoints(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), x, y)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, - Extent.fromPoints(x - randomIntBetween(1, 10), y - randomIntBetween(1, 10), - x + randomIntBetween(1, 10), y + randomIntBetween(1, 10))); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, - Extent.fromPoints(x - randomIntBetween(10, 100), y - randomIntBetween(10, 100), - x - randomIntBetween(1, 10), y - randomIntBetween(1, 10))); - } - - public void testPoints() throws IOException { - for (int i = 0; i < 500; i++) { - int minX = randomIntBetween(-180, 170); - int maxX = randomIntBetween(minX + 10, 180); - int minY = randomIntBetween(-90, 80); - int maxY = randomIntBetween(minY + 10, 90); - Extent extent = Extent.fromPoints(minX, minY, maxX, maxY); - int numPoints = randomIntBetween(2, 1000); - - double[] x = new double[numPoints]; - double[] y = new double[numPoints]; - for (int j = 0; j < numPoints; j++) { - x[j] = randomIntBetween(minX, maxX); - y[j] = randomIntBetween(minY, maxY); - } - Point2DWriter writer = new Point2DWriter(x, y, TestCoordinateEncoder.INSTANCE); - - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - Point2DReader reader = new Point2DReader(new ByteBufferStreamInput(ByteBuffer.wrap(output.bytes().toBytesRef().bytes))); - // tests calling getExtent() and relate() multiple times to make sure deserialization is not affected - assertThat(reader.getExtent(), equalTo(reader.getExtent())); - assertThat(reader.getExtent(), equalTo(writer.getExtent())); - assertRelation(GeoRelation.QUERY_CROSSES, reader, extent); - assertRelation(GeoRelation.QUERY_CROSSES, reader, extent); - } - } -} diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java index c0d9c76d18298..99c24fe1b6298 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -18,21 +18,289 @@ */ package org.elasticsearch.common.geo; -import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; +import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; +import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; +import static org.elasticsearch.geo.GeometryTestUtils.fold; +import static org.elasticsearch.geo.GeometryTestUtils.randomPoint; +import static org.hamcrest.Matchers.equalTo; -public class TriangleTreeTests extends AbstractTreeTestCase { +public class TriangleTreeTests extends ESTestCase { - protected ShapeTreeReader geometryTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { - TriangleTreeWriter writer = new TriangleTreeWriter(geometry, encoder); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - TriangleTreeReader reader = new TriangleTreeReader(encoder); - reader.reset(output.bytes().toBytesRef()); - return reader; + public void testRectangleShape() throws IOException { + for (int i = 0; i < 1000; i++) { + int minX = randomIntBetween(-80, 70); + int maxX = randomIntBetween(minX + 10, 80); + int minY = randomIntBetween(-80, 70); + int maxY = randomIntBetween(minY + 10, 80); + double[] x = new double[]{minX, maxX, maxX, minX, minX}; + double[] y = new double[]{minY, minY, maxY, maxY, minY}; + Geometry rectangle = randomBoolean() ? + new Polygon(new LinearRing(x, y), Collections.emptyList()) : new Rectangle(minX, maxX, maxY, minY); + TriangleTreeReader reader = triangleTreeReader(rectangle, TestCoordinateEncoder.INSTANCE); + + assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); + // encoder loses precision when casting to integer, so centroid is calculated using integer division here + assertThat(reader.getCentroidX(), equalTo((double) ((minX + maxX) / 2))); + assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY) / 2))); + + // box-query touches bottom-left corner + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), + minY - randomIntBetween(1, 180), minX, minY)); + // box-query touches bottom-right corner + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), + maxX + randomIntBetween(1, 180), minY)); + // box-query touches top-right corner + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), + maxY + randomIntBetween(1, 180))); + // box-query touches top-left corner + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, + maxY + randomIntBetween(1, 180))); + // box-query fully-enclosed inside rectangle + assertRelation(GeoRelation.QUERY_INSIDE,reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); + // box-query fully-contains poly + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), + minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + // box-query half-in-half-out-right + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + // box-query half-in-half-out-left + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, + (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); + // box-query half-in-half-out-top + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, + maxX + randomIntBetween(1, 1000), maxY + randomIntBetween(1, 1000))); + // box-query half-in-half-out-bottom + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), + maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + + // box-query outside to the right + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, + maxX + randomIntBetween(1001, 2000), maxY)); + // box-query outside to the left + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, + minX - randomIntBetween(1, 1000), maxY)); + // box-query outside to the top + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, + maxY + randomIntBetween(1001, 2000))); + // box-query outside to the bottom + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, + minY - randomIntBetween(1, 1000))); + } + } + + public void testPacManPolygon() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // test cell crossing poly + TriangleTreeReader reader = triangleTreeReader(new Polygon(new LinearRing(py, px), Collections.emptyList()), + TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(-5, -6, 2, -2)); + } + + // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon + public void testPolygonWithHole() throws Exception { + Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), + Collections.singletonList(new LinearRing(new double[]{-10, 10, 10, -10, -10}, new double[]{-10, -10, 10, 10, -10}))); + + TriangleTreeReader reader = triangleTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); + + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(6, -6, 6, -6)); // in the hole + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(25, -25, 25, -25)); // on the mainland + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(51, 51, 52, 52)); // outside of mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-60, -60, 60, 60)); // enclosing us completely + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(49, 49, 51, 51)); // overlapping the mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(9, 9, 11, 11)); // overlapping the hole + } + + public void testCombPolygon() throws Exception { + double[] px = {0, 10, 10, 20, 20, 30, 30, 40, 40, 50, 50, 0, 0}; + double[] py = {0, 0, 20, 20, 0, 0, 20, 20, 0, 0, 30, 30, 0}; + + double[] hx = {21, 21, 29, 29, 21}; + double[] hy = {1, 20, 20, 1, 1}; + + Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); + TriangleTreeReader reader = triangleTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); + // test cell crossing poly + assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(5, 10, 5, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(15, 10, 15, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(25, 10, 25, 10)); + } + + public void testPacManClosedLineString() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + + // test cell crossing poly + TriangleTreeReader reader = triangleTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); + } + + public void testPacManLineString() throws Exception { + // pacman + double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10}; + double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; + + // test cell crossing poly + TriangleTreeReader reader = triangleTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); + } + + public void testPacManPoints() throws Exception { + // pacman + List points = Arrays.asList( + new Point(0, 0), + new Point(5, 10), + new Point(9, 10), + new Point(10, 0), + new Point(9, -8), + new Point(0, -10), + new Point(-9, -8), + new Point(-10, 0), + new Point(-9, 10), + new Point(-5, 10) + ); + + + // candidate intersects cell + int xMin = 0; + int xMax = 11; + int yMin = -10; + int yMax = 9; + + // test cell crossing poly + TriangleTreeReader reader = triangleTreeReader(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(xMin, yMin, xMax, yMax)); + } + + public void testRandomMultiLineIntersections() throws IOException { + double extentSize = randomDoubleBetween(0.01, 10, true); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + MultiLine geometry = GeometryTestUtils.randomMultiLine(false); + geometry = (MultiLine) indexer.prepareForIndexing(geometry); + + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + Extent readerExtent = reader.getExtent(); + + for (Line line : geometry) { + // extent that intersects edges + assertRelation(GeoRelation.QUERY_CROSSES, reader, bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize)); + + // extent that fully encloses a line in the MultiLine + Extent lineExtent = triangleTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); + assertRelation(GeoRelation.QUERY_CROSSES, reader, lineExtent); + + if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE + && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, + lineExtent.maxX() + 1, lineExtent.maxY() + 1)); + } + } + + // extent that fully encloses the MultiLine + assertRelation(GeoRelation.QUERY_CROSSES, reader, reader.getExtent()); + if (readerExtent.minX() != Integer.MIN_VALUE && readerExtent.maxX() != Integer.MAX_VALUE + && readerExtent.minY() != Integer.MIN_VALUE && readerExtent.maxY() != Integer.MAX_VALUE) { + assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(readerExtent.minX() - 1, readerExtent.minY() - 1, + readerExtent.maxX() + 1, readerExtent.maxY() + 1)); + } + + } + + public void testRandomGeometryIntersection() throws IOException { + int testPointCount = randomIntBetween(100, 200); + Point[] testPoints = new Point[testPointCount]; + double extentSize = randomDoubleBetween(1, 10, true); + boolean[] intersects = new boolean[testPointCount]; + for (int i = 0; i < testPoints.length; i++) { + testPoints[i] = randomPoint(false); + } + + Geometry geometry = randomGeometryTreeGeometry(); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry preparedGeometry = indexer.prepareForIndexing(geometry); + + for (int i = 0; i < testPointCount; i++) { + int cur = i; + intersects[cur] = fold(preparedGeometry, false, (g, s) -> s || intersects(g, testPoints[cur], extentSize)); + } + + for (int i = 0; i < testPointCount; i++) { + assertEquals(intersects[i], intersects(preparedGeometry, testPoints[i], extentSize)); + } + } + + private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) { + int xMin = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.max(x - extentSize, -180.0)); + int xMax = GeoShapeCoordinateEncoder.INSTANCE.encodeX(Math.min(x + extentSize, 180.0)); + int yMin = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.max(y - extentSize, -90)); + int yMax = GeoShapeCoordinateEncoder.INSTANCE.encodeY(Math.min(y + extentSize, 90)); + return Extent.fromPoints(xMin, yMin, xMax, yMax); + } + + private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { + + Extent bufferBounds = bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize); + GeoRelation relation = triangleTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) + .relate(bufferBounds.minX(), bufferBounds.minY(), bufferBounds.maxX(), bufferBounds.maxY()); + return relation == GeoRelation.QUERY_CROSSES || relation == GeoRelation.QUERY_INSIDE; + } + + private static Geometry randomGeometryTreeGeometry() { + return randomGeometryTreeGeometry(0); + } + + private static Geometry randomGeometryTreeGeometry(int level) { + @SuppressWarnings("unchecked") Function geometry = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPoint, + level < 3 ? (b) -> randomGeometryTreeCollection(level + 1) : GeometryTestUtils::randomPoint // don't build too deep + ); + return geometry.apply(false); + } + + private static Geometry randomGeometryTreeCollection(int level) { + int size = ESTestCase.randomIntBetween(1, 10); + List shapes = new ArrayList<>(); + for (int i = 0; i < size; i++) { + shapes.add(randomGeometryTreeGeometry(level)); + } + return new GeometryCollection<>(shapes); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java index 5463a3af08d74..6d270b8ec18f9 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -19,9 +19,7 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; -import org.elasticsearch.common.geo.GeometryTreeReader; -import org.elasticsearch.common.geo.GeometryTreeWriter; -import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.geo.TriangleTreeReader; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.MultiLine; @@ -33,7 +31,7 @@ import java.util.Arrays; -import static org.elasticsearch.common.geo.GeoTestUtils.geometryTreeReader; +import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; import static org.hamcrest.Matchers.equalTo; public class GeoGridTilerTests extends ESTestCase { @@ -48,14 +46,9 @@ public void testGeoTile() throws Exception { // create rectangle within tile and check bound counts Rectangle tile = GeoTileUtils.toBoundingBox(1309, 3166, 13); - GeometryTreeWriter writer = new GeometryTreeWriter( - new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, - tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001), GeoShapeCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); - reader.reset(output.bytes().toBytesRef()); + Rectangle shapeRectangle = new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, + tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001); + TriangleTreeReader reader = triangleTreeReader(shapeRectangle, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); // test shape within tile bounds @@ -98,7 +91,7 @@ private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Ex int precision = randomIntBetween(1, 10); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); geometry = indexer.prepareForIndexing(geometry); - GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); CellIdSource.GeoShapeCellValues recursiveValues = new CellIdSource.GeoShapeCellValues(null, precision, GEOTILE); int recursiveCount; @@ -129,7 +122,7 @@ private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Ex int precision = randomIntBetween(1, 4); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); geometry = indexer.prepareForIndexing(geometry); - GeometryTreeReader reader = geometryTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); CellIdSource.GeoShapeCellValues recursiveValues = new CellIdSource.GeoShapeCellValues(null, precision, GEOHASH); int recursiveCount; @@ -158,14 +151,9 @@ public void testGeoHash() throws Exception { Rectangle tile = Geohash.toBoundingBox(Geohash.stringEncode(x, y, 5)); - GeometryTreeWriter writer = new GeometryTreeWriter( - new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, - tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001), GeoShapeCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); - writer.writeTo(output); - output.close(); - GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); - reader.reset(output.bytes().toBytesRef()); + Rectangle shapeRectangle = new Rectangle(tile.getMinX() + 0.00001, tile.getMaxX() - 0.00001, + tile.getMaxY() - 0.00001, tile.getMinY() + 0.00001); + TriangleTreeReader reader = triangleTreeReader(shapeRectangle, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); // test shape within tile bounds diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java index 4209fe933c100..aa98274115356 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java @@ -209,8 +209,6 @@ public void testSingleValuedFieldNearDateLine() throws Exception { } public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { - GeoPoint geoValuesTopLeft = new GeoPoint(38, 170); - GeoPoint geoValuesBottomRight = new GeoPoint(-24, -175); SearchResponse response = client().prepareSearch(DATELINE_IDX_NAME) .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME).wrapLongitude(true)) .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME).wrapLongitude(true)) @@ -218,10 +216,28 @@ public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { assertSearchResponse(response); - for (String aggName : List.of(geoPointAggName, geoShapeAggName)) { - GeoBounds geoBounds = response.getAggregations().get(aggName); + // test geo_point + { + GeoPoint geoValuesTopLeft = new GeoPoint(38, 170); + GeoPoint geoValuesBottomRight = new GeoPoint(-24, -175); + GeoBounds geoBounds = response.getAggregations().get(geoPointAggName); assertThat(geoBounds, notNullValue()); - assertThat(geoBounds.getName(), equalTo(aggName)); + assertThat(geoBounds.getName(), equalTo(geoPointAggName)); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(geoValuesTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(geoValuesBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); + } + + // test geo_shape + { + GeoPoint geoValuesTopLeft = new GeoPoint(38, 178); + GeoPoint geoValuesBottomRight = new GeoPoint(-24, -179); + GeoBounds geoBounds = response.getAggregations().get(geoShapeAggName); + assertThat(geoBounds, notNullValue()); + assertThat(geoBounds.getName(), equalTo(geoShapeAggName)); GeoPoint topLeft = geoBounds.topLeft(); GeoPoint bottomRight = geoBounds.bottomRight(); assertThat(topLeft.lat(), closeTo(geoValuesTopLeft.lat(), GEOHASH_TOLERANCE)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index f9f5c995f7d40..04edd9172eb26 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -28,10 +28,8 @@ import org.apache.lucene.util.TestUtil; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; -import org.elasticsearch.common.geo.GeometryTreeReader; -import org.elasticsearch.common.geo.TriangleTreeWriter; +import org.elasticsearch.common.geo.TriangleTreeReader; import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.MultiGeoValues; @@ -46,6 +44,8 @@ import java.util.Set; import java.util.function.LongUnaryOperator; +import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; + public class MissingValuesTests extends ESTestCase { public void testMissingBytes() throws IOException { @@ -406,15 +406,11 @@ public ValuesSourceType valuesSourceType() { public void testMissingGeoShapes() throws IOException { final int numDocs = TestUtil.nextInt(random(), 1, 100); final MultiGeoValues.GeoShapeValue[][] values = new MultiGeoValues.GeoShapeValue[numDocs][]; - GeometryTreeReader reader = new GeometryTreeReader(GeoShapeCoordinateEncoder.INSTANCE); for (int i = 0; i < numDocs; ++i) { values[i] = new MultiGeoValues.GeoShapeValue[random().nextInt(4)]; for (int j = 0; j < values[i].length; ++j) { ShapeBuilder builder = RandomShapeGenerator.createShape(random()); - BytesStreamOutput outputStream = new BytesStreamOutput(); - TriangleTreeWriter writer = new TriangleTreeWriter(builder.buildGeometry(), GeoShapeCoordinateEncoder.INSTANCE); - writer.writeTo(outputStream); - reader.reset(outputStream.bytes().toBytesRef()); + TriangleTreeReader reader = triangleTreeReader(builder.buildGeometry(), GeoShapeCoordinateEncoder.INSTANCE); values[i][j] = new MultiGeoValues.GeoShapeValue(reader); } } @@ -446,10 +442,7 @@ public ValuesSourceType valuesSourceType() { } }; ShapeBuilder builder = RandomShapeGenerator.createShape(random()); - BytesStreamOutput outputStream = new BytesStreamOutput(); - TriangleTreeWriter writer = new TriangleTreeWriter(builder.buildGeometry(), GeoShapeCoordinateEncoder.INSTANCE); - writer.writeTo(outputStream); - reader.reset(outputStream.bytes().toBytesRef()); + TriangleTreeReader reader = triangleTreeReader(builder.buildGeometry(), GeoShapeCoordinateEncoder.INSTANCE); final MultiGeoValues.GeoShapeValue missing = new MultiGeoValues.GeoShapeValue(reader); MultiGeoValues withMissingReplaced = MissingValues.replaceMissing(asGeoValues, missing); for (int i = 0; i < numDocs; ++i) { From 6f5919aa1910d806d7a8bbf377eef46ce7f780dc Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 12 Dec 2019 07:53:11 +0100 Subject: [PATCH 45/62] Build triangle tree from geo_shape indexed fields (#50012) --- .../common/geo/CentroidCalculator.java | 104 +++- .../org/elasticsearch/common/geo/Extent.java | 51 +- .../common/geo/ShapeTreeWriter.java | 34 -- .../common/geo/TriangleTreeReader.java | 149 ++--- .../common/geo/TriangleTreeWriter.java | 566 ++++-------------- .../index/fielddata/MultiGeoValues.java | 92 +-- .../mapper/AbstractGeometryFieldMapper.java | 20 +- .../mapper/BinaryGeoShapeDocValuesField.java | 34 +- .../index/mapper/GeoShapeIndexer.java | 13 +- .../index/mapper/LegacyGeoShapeIndexer.java | 4 +- .../common/geo/CentroidCalculatorTests.java | 33 +- .../elasticsearch/common/geo/ExtentTests.java | 25 +- .../common/geo/GeoTestUtils.java | 32 +- .../common/geo/TriangleTreeTests.java | 141 +++-- .../geogrid/GeoGridAggregatorTestCase.java | 15 +- .../metrics/GeoBoundsAggregatorTests.java | 10 +- .../aggregations/metrics/GeoBoundsIT.java | 6 +- .../metrics/GeoCentroidAggregatorTests.java | 100 +--- .../spatial/index/mapper/ShapeIndexer.java | 9 +- 19 files changed, 563 insertions(+), 875 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java index 998f2753420ff..51b610c02f431 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java +++ b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java @@ -19,6 +19,19 @@ package org.elasticsearch.common.geo; +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; + /** * This class keeps a running Kahan-sum of coordinates * that are to be averaged in {@link TriangleTreeWriter} for use @@ -32,12 +45,13 @@ public class CentroidCalculator { private double sumY; private int count; - public CentroidCalculator() { + public CentroidCalculator(Geometry geometry) { this.sumX = 0.0; this.compX = 0.0; this.sumY = 0.0; this.compY = 0.0; this.count = 0; + geometry.visit(new CentroidCalculatorVisitor(this)); } /** @@ -47,7 +61,7 @@ public CentroidCalculator() { * @param x the x-coordinate of the point * @param y the y-coordinate of the point */ - public void addCoordinate(double x, double y) { + private void addCoordinate(double x, double y) { double correctedX = x - compX; double newSumX = sumX + correctedX; compX = (newSumX - sumX) - correctedX; @@ -69,7 +83,7 @@ public void addCoordinate(double x, double y) { * * @param otherCalculator the other centroid calculator to add from */ - void addFrom(CentroidCalculator otherCalculator) { + public void addFrom(CentroidCalculator otherCalculator) { addCoordinate(otherCalculator.sumX, otherCalculator.sumY); // adjust count count += otherCalculator.count - 1; @@ -88,4 +102,88 @@ public double getX() { public double getY() { return sumY / count; } + + private static class CentroidCalculatorVisitor implements GeometryVisitor { + + private final CentroidCalculator calculator; + + private CentroidCalculatorVisitor(CentroidCalculator calculator) { + this.calculator = calculator; + } + + @Override + public Void visit(Circle circle) { + calculator.addCoordinate(circle.getX(), circle.getY()); + return null; + } + + @Override + public Void visit(GeometryCollection collection) { + for (Geometry shape : collection) { + shape.visit(this); + } + return null; + } + + @Override + public Void visit(Line line) { + + for (int i = 0; i < line.length(); i++) { + calculator.addCoordinate(line.getX(i), line.getY(i)); + } + return null; + } + + @Override + public Void visit(LinearRing ring) { + for (int i = 0; i < ring.length() - 1; i++) { + calculator.addCoordinate(ring.getX(i), ring.getY(i)); + } + return null; + } + + @Override + public Void visit(MultiLine multiLine) { + for (Line line : multiLine) { + visit(line); + } + return null; + } + + @Override + public Void visit(MultiPoint multiPoint) { + for (Point point : multiPoint) { + visit(point); + } + return null; + } + + @Override + public Void visit(MultiPolygon multiPolygon) { + for (Polygon polygon : multiPolygon) { + visit(polygon); + } + return null; + } + + @Override + public Void visit(Point point) { + calculator.addCoordinate(point.getX(), point.getY()); + return null; + } + + @Override + public Void visit(Polygon polygon) { + return visit(polygon.getPolygon()); + } + + @Override + public Void visit(Rectangle rectangle) { + calculator.addCoordinate(rectangle.getMinX(), rectangle.getMinY()); + calculator.addCoordinate(rectangle.getMinX(), rectangle.getMaxY()); + calculator.addCoordinate(rectangle.getMaxX(), rectangle.getMinY()); + calculator.addCoordinate(rectangle.getMaxX(), rectangle.getMaxY()); + return null; + } + } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index 2c3eef24367a5..eec3d7ca408cd 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -18,18 +18,12 @@ */ package org.elasticsearch.common.geo; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; import java.util.Objects; /** - * Object representing the extent of a geometry object within a {@link ShapeTreeWriter}. + * Object representing the extent of a geometry object within a {@link TriangleTreeWriter}. */ -public class Extent implements Writeable { - static final int WRITEABLE_SIZE_IN_BYTES = 24; +public class Extent { public int top; public int bottom; @@ -47,7 +41,7 @@ public Extent() { this.posRight = Integer.MIN_VALUE; } - public Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + private Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { this.top = top; this.bottom = bottom; this.negLeft = negLeft; @@ -56,10 +50,6 @@ public Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int p this.posRight = posRight; } - Extent(StreamInput input) throws IOException { - this(input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt(), input.readInt()); - } - public void reset(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { this.top = top; this.bottom = bottom; @@ -88,9 +78,10 @@ public void addRectangle(int bottomLeftX, int bottomLeftY, int topRightX, int to this.negRight = Math.max(this.negRight, topRightX); } else if (bottomLeftX < 0) { this.negLeft = Math.min(this.negLeft, bottomLeftX); - this.negRight = Math.max(this.negRight, bottomLeftX); - this.posLeft = Math.min(this.posLeft, topRightX); this.posRight = Math.max(this.posRight, topRightX); + // this signal the extent cannot be wrapped around the dateline + this.negRight = 0; + this.posLeft = 0; } else { this.posLeft = Math.min(this.posLeft, bottomLeftX); this.posRight = Math.max(this.posRight, topRightX); @@ -131,8 +122,11 @@ static Extent fromPoints(int bottomLeftX, int bottomLeftY, int topRightX, int to negLeft = bottomLeftX; negRight = topRightX; } else if (bottomLeftX < 0) { - negLeft = negRight = bottomLeftX; - posLeft = posRight = topRightX; + negLeft = bottomLeftX; + posRight = topRightX; + // this signal the extent cannot be wrapped around the dateline + negRight = 0; + posLeft = 0; } else { posLeft = bottomLeftX; posRight = topRightX; @@ -168,17 +162,6 @@ public int maxX() { return Math.max(negRight, posRight); } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeInt(top); - out.writeInt(bottom); - out.writeInt(negLeft); - out.writeInt(negRight); - out.writeInt(posLeft); - out.writeInt(posRight); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -196,4 +179,16 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(top, bottom, negLeft, negRight, posLeft, posRight); } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("["); + builder.append("top = " + top + ", "); + builder.append("bottom = " + bottom + ", "); + builder.append("negLeft = " + negLeft + ", "); + builder.append("negRight = " + negRight + ", "); + builder.append("posLeft = " + posLeft + ", "); + builder.append("posRight = " + posRight + "]"); + return builder.toString(); + } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java deleted file mode 100644 index 35eaec5fb02f0..0000000000000 --- a/server/src/main/java/org/elasticsearch/common/geo/ShapeTreeWriter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.common.geo; - -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.geometry.ShapeType; - -/** - * Shape writer for use in doc-values - */ -public abstract class ShapeTreeWriter implements Writeable { - - public abstract Extent getExtent(); - - public abstract ShapeType getShapeType(); - - public abstract CentroidCalculator getCentroidCalculator(); -} diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index b44bc3cb687ef..4b346a1302cdf 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -18,11 +18,10 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.io.stream.ByteBufferStreamInput; import java.io.IOException; -import java.nio.ByteBuffer; import static org.apache.lucene.geo.GeoUtils.orient; @@ -35,8 +34,8 @@ */ public class TriangleTreeReader { - private final int extentOffset = 8; - private ByteBufferStreamInput input; + private static final int extentOffset = 8; + private final ByteArrayDataInput input; private final CoordinateEncoder coordinateEncoder; private final Rectangle2D rectangle2D; private final Extent extent; @@ -46,19 +45,21 @@ public TriangleTreeReader(CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; this.rectangle2D = new Rectangle2D(); this.extent = new Extent(); + this.input = new ByteArrayDataInput(); } public void reset(BytesRef bytesRef) throws IOException { - this.input = new ByteBufferStreamInput(ByteBuffer.wrap(bytesRef.bytes, bytesRef.offset, bytesRef.length)); + this.input.reset(bytesRef.bytes, bytesRef.offset, bytesRef.length); treeOffset = 0; } /** * returns the bounding box of the geometry in the format [minX, maxX, minY, maxY]. */ - public Extent getExtent() throws IOException { + public Extent getExtent() { if (treeOffset == 0) { - input.position(extentOffset); + // TODO: Compress serialization of extent + input.setPosition(extentOffset); int top = input.readInt(); int bottom = Math.toIntExact(top - input.readVLong()); int posRight = input.readInt(); @@ -66,9 +67,9 @@ public Extent getExtent() throws IOException { int negRight = input.readInt(); int negLeft = input.readInt(); extent.reset(top, bottom, negLeft, negRight, posLeft, posRight); - treeOffset = input.position(); + treeOffset = input.getPosition(); } else { - input.position(treeOffset); + input.setPosition(treeOffset); } return extent; } @@ -76,16 +77,16 @@ public Extent getExtent() throws IOException { /** * returns the X coordinate of the centroid. */ - public double getCentroidX() throws IOException { - input.position(0); + public double getCentroidX() { + input.setPosition(0); return coordinateEncoder.decodeX(input.readInt()); } /** * returns the Y coordinate of the centroid. */ - public double getCentroidY() throws IOException { - input.position(4); + public double getCentroidY() { + input.setPosition(4); return coordinateEncoder.decodeY(input.readInt()); } @@ -93,76 +94,79 @@ public double getCentroidY() throws IOException { * Compute the relation with the provided bounding box. If the result is CELL_INSIDE_QUERY * then the bounding box is within the shape. */ - public GeoRelation relate(int minX, int minY, int maxX, int maxY) throws IOException { + public GeoRelation relate(int minX, int minY, int maxX, int maxY) { Extent extent = getExtent(); int thisMaxX = extent.maxX(); int thisMinX = extent.minX(); int thisMaxY = extent.maxY(); int thisMinY = extent.minY(); + if ((thisMinX > maxX || thisMaxX < minX || thisMinY > maxY || thisMaxY < minY)) { + // shapes are disjoint + return GeoRelation.QUERY_DISJOINT; + } if (minX <= thisMinX && maxX >= thisMaxX && minY <= thisMinY && maxY >= thisMaxY) { // the rectangle fully contains the shape return GeoRelation.QUERY_CROSSES; } + // quick checks failed, need to traverse the tree GeoRelation rel = GeoRelation.QUERY_DISJOINT; - if ((thisMinX > maxX || thisMaxX < minX || thisMinY > maxY || thisMaxY < minY) == false) { - // shapes are NOT disjoint - rectangle2D.setValues(minX, maxX, minY, maxY); - byte metadata = input.readByte(); - if ((metadata & 1 << 2) == 1 << 2) { // component in this node is a point - int x = Math.toIntExact(thisMaxX - input.readVLong()); - int y = Math.toIntExact(thisMaxY - input.readVLong()); - if (rectangle2D.contains(x, y)) { - return GeoRelation.QUERY_CROSSES; - } - thisMinX = x; - } else if ((metadata & 1 << 3) == 1 << 3) { // component in this node is a line - int aX = Math.toIntExact(thisMaxX - input.readVLong()); - int aY = Math.toIntExact(thisMaxY - input.readVLong()); - int bX = Math.toIntExact(thisMaxX - input.readVLong()); - int bY = Math.toIntExact(thisMaxY - input.readVLong()); - if (rectangle2D.intersectsLine(aX, aY, bX, bY)) { - return GeoRelation.QUERY_CROSSES; - } - thisMinX = aX; - } else { // component in this node is a triangle - int aX = Math.toIntExact(thisMaxX - input.readVLong()); - int aY = Math.toIntExact(thisMaxY - input.readVLong()); - int bX = Math.toIntExact(thisMaxX - input.readVLong()); - int bY = Math.toIntExact(thisMaxY - input.readVLong()); - int cX = Math.toIntExact(thisMaxX - input.readVLong()); - int cY = Math.toIntExact(thisMaxY - input.readVLong()); - boolean ab = (metadata & 1 << 4) == 1 << 4; - boolean bc = (metadata & 1 << 5) == 1 << 5; - boolean ca = (metadata & 1 << 6) == 1 << 6; - rel = rectangle2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); - if (rel == GeoRelation.QUERY_CROSSES) { - return GeoRelation.QUERY_CROSSES; - } - thisMinX = aX; + rectangle2D.setValues(minX, maxX, minY, maxY); + byte metadata = input.readByte(); + if ((metadata & 1 << 2) == 1 << 2) { // component in this node is a point + int x = Math.toIntExact(thisMaxX - input.readVLong()); + int y = Math.toIntExact(thisMaxY - input.readVLong()); + if (rectangle2D.contains(x, y)) { + return GeoRelation.QUERY_CROSSES; } - if ((metadata & 1 << 0) == 1 << 0) { // left != null - GeoRelation left = relate(rectangle2D, false, thisMaxX, thisMaxY); - if (left == GeoRelation.QUERY_CROSSES) { - return GeoRelation.QUERY_CROSSES; - } else if (left == GeoRelation.QUERY_INSIDE) { - rel = left; - } + thisMinX = x; + } else if ((metadata & 1 << 3) == 1 << 3) { // component in this node is a line + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + if (rectangle2D.intersectsLine(aX, aY, bX, bY)) { + return GeoRelation.QUERY_CROSSES; } - if ((metadata & 1 << 1) == 1 << 1) { // right != null - if (rectangle2D.maxX >= thisMinX) { - GeoRelation right = relate(rectangle2D, false, thisMaxX, thisMaxY); - if (right == GeoRelation.QUERY_CROSSES) { - return GeoRelation.QUERY_CROSSES; - } else if (right == GeoRelation.QUERY_INSIDE) { - rel = right; - } + thisMinX = aX; + } else { // component in this node is a triangle + int aX = Math.toIntExact(thisMaxX - input.readVLong()); + int aY = Math.toIntExact(thisMaxY - input.readVLong()); + int bX = Math.toIntExact(thisMaxX - input.readVLong()); + int bY = Math.toIntExact(thisMaxY - input.readVLong()); + int cX = Math.toIntExact(thisMaxX - input.readVLong()); + int cY = Math.toIntExact(thisMaxY - input.readVLong()); + boolean ab = (metadata & 1 << 4) == 1 << 4; + boolean bc = (metadata & 1 << 5) == 1 << 5; + boolean ca = (metadata & 1 << 6) == 1 << 6; + rel = rectangle2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); + if (rel == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } + thisMinX = aX; + } + if ((metadata & 1 << 0) == 1 << 0) { // left != null + GeoRelation left = relate(rectangle2D, false, thisMaxX, thisMaxY); + if (left == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (left == GeoRelation.QUERY_INSIDE) { + rel = left; + } + } + if ((metadata & 1 << 1) == 1 << 1) { // right != null + if (rectangle2D.maxX >= thisMinX) { + GeoRelation right = relate(rectangle2D, false, thisMaxX, thisMaxY); + if (right == GeoRelation.QUERY_CROSSES) { + return GeoRelation.QUERY_CROSSES; + } else if (right == GeoRelation.QUERY_INSIDE) { + rel = right; } } } + return rel; } - private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMaxX, int parentMaxY) throws IOException { + private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMaxX, int parentMaxY) { int thisMaxX = Math.toIntExact(parentMaxX - input.readVLong()); int thisMaxY = Math.toIntExact(parentMaxY - input.readVLong()); GeoRelation rel = GeoRelation.QUERY_DISJOINT; @@ -224,11 +228,11 @@ private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMa rel = right; } } else { - input.skip(rightSize); + input.skipBytes(rightSize); } } } else { - input.skip(size); + input.skipBytes(size); } return rel; } @@ -243,7 +247,7 @@ private static class Rectangle2D { Rectangle2D() { } - protected void setValues(int minX, int maxX, int minY, int maxY) { + private void setValues(int minX, int maxX, int minY, int maxY) { this.minX = minX; this.maxX = maxX; this.minY = minY; @@ -260,7 +264,7 @@ public boolean contains(int x, int y) { /** * Checks if the rectangle intersects the provided triangle **/ - public boolean intersectsLine(int aX, int aY, int bX, int bY) { + private boolean intersectsLine(int aX, int aY, int bX, int bY) { // 1. query contains any triangle points if (contains(aX, aY) || contains(bX, bY)) { return true; @@ -287,7 +291,7 @@ public boolean intersectsLine(int aX, int aY, int bX, int bY) { /** * Checks if the rectangle intersects the provided triangle **/ - public GeoRelation relateTriangle(int aX, int aY, boolean ab, int bX, int bY, boolean bc, int cX, int cY, boolean ca) { + private GeoRelation relateTriangle(int aX, int aY, boolean ab, int bX, int bY, boolean bc, int cX, int cY, boolean ca) { // 1. query contains any triangle points if (contains(aX, aY) || contains(bX, bY) || contains(cX, cY)) { return GeoRelation.QUERY_CROSSES; @@ -374,8 +378,8 @@ private boolean edgeIntersectsQuery(int ax, int ay, int bx, int by) { /** * Compute whether the given x, y point is in a triangle; uses the winding order method */ - static boolean pointInTriangle(double minX, double maxX, double minY, double maxY, double x, double y, - double aX, double aY, double bX, double bY, double cX, double cY) { + private static boolean pointInTriangle(double minX, double maxX, double minY, double maxY, double x, double y, + double aX, double aY, double bX, double bY, double cX, double cY) { //check the bounding box because if the triangle is degenerated, e.g points and lines, we need to filter out //coplanar points that are not part of the triangle. if (x >= minX && x <= maxX && y >= minY && y <= maxY) { @@ -398,6 +402,5 @@ private static boolean boxesAreDisjoint(final int aMinX, final int aMaxX, final final int bMinX, final int bMaxX, final int bMinY, final int bMaxY) { return (aMaxX < bMinX || aMinX > bMaxX || aMaxY < bMinY || aMinY > bMaxY); } - } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java index f1cca92c303f3..f4efa0523c7f1 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java @@ -18,59 +18,38 @@ */ package org.elasticsearch.common.geo; -import org.apache.lucene.geo.GeoUtils; -import org.apache.lucene.geo.Tessellator; +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.util.ArrayUtil; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.geometry.Circle; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.GeometryCollection; -import org.elasticsearch.geometry.GeometryVisitor; -import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; -import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.ShapeType; -import org.elasticsearch.index.mapper.GeoShapeIndexer; import java.io.IOException; -import java.util.ArrayList; import java.util.Comparator; import java.util.List; /** - * This is a tree-writer that serializes a {@link Geometry} and tessellate it to write it into a byte array. - * Internally it tessellate the given {@link Geometry} and it builds an interval tree with the - * tessellation. + * This is a tree-writer that serializes a list of {@link ShapeField.DecodedTriangle} as an interval tree + * into a byte array. */ -public class TriangleTreeWriter extends ShapeTreeWriter { +public class TriangleTreeWriter { private final TriangleTreeNode node; private final CoordinateEncoder coordinateEncoder; private final CentroidCalculator centroidCalculator; - private final ShapeType type; private Extent extent; - public TriangleTreeWriter(Geometry geometry, CoordinateEncoder coordinateEncoder) { + public TriangleTreeWriter(List triangles, CoordinateEncoder coordinateEncoder, + CentroidCalculator centroidCalculator) { this.coordinateEncoder = coordinateEncoder; - this.centroidCalculator = new CentroidCalculator(); + this.centroidCalculator = centroidCalculator; this.extent = new Extent(); - this.type = geometry.type(); - TriangleTreeBuilder builder = new TriangleTreeBuilder(coordinateEncoder); - geometry.visit(builder); - this.node = builder.build(); + this.node = build(triangles); } - @Override - public void writeTo(StreamOutput out) throws IOException { + /*** Serialize the interval tree in the provided data output */ + public void writeTo(ByteBuffersDataOutput out) throws IOException { out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); + // TODO: Compress serialization of extent out.writeInt(extent.top); out.writeVLong((long) extent.top - extent.bottom); out.writeInt(extent.posRight); @@ -80,218 +59,105 @@ public void writeTo(StreamOutput out) throws IOException { node.writeTo(out); } - @Override - public Extent getExtent() { - return extent; + private void addToExtent(TriangleTreeNode treeNode) { + extent.addRectangle(treeNode.minX, treeNode.minY, treeNode.maxX, treeNode.maxY); } - @Override - public ShapeType getShapeType() { - return type; - } - - @Override - public CentroidCalculator getCentroidCalculator() { - return centroidCalculator; - } - - /** - * Class that tessellate the geometry and build an interval tree in memory. - */ - class TriangleTreeBuilder implements GeometryVisitor { - - private final List triangles; - private final CoordinateEncoder coordinateEncoder; - - - TriangleTreeBuilder(CoordinateEncoder coordinateEncoder) { - this.coordinateEncoder = coordinateEncoder; - this.triangles = new ArrayList<>(); + private TriangleTreeNode build(List triangles) { + if (triangles.size() == 1) { + TriangleTreeNode triangleTreeNode = new TriangleTreeNode(triangles.get(0)); + addToExtent(triangleTreeNode); + return triangleTreeNode; } - - private void addTriangles(List triangles) { - this.triangles.addAll(triangles); - } - - @Override - public Void visit(GeometryCollection collection) { - for (Geometry geometry : collection) { - geometry.visit(this); - } - return null; - } - - @Override - public Void visit(Line line) { - for (int i = 0; i < line.length(); i++) { - centroidCalculator.addCoordinate(line.getX(i), line.getY(i)); - } - org.apache.lucene.geo.Line luceneLine = GeoShapeIndexer.toLuceneLine(line); - addToExtent(luceneLine.minLon, luceneLine.maxLon, luceneLine.minLat, luceneLine.maxLat); - addTriangles(TriangleTreeLeaf.fromLine(coordinateEncoder, luceneLine)); - return null; - } - - @Override - public Void visit(MultiLine multiLine) { - for (Line line : multiLine) { - visit(line); - } - return null; - } - - @Override - public Void visit(Polygon polygon) { - // TODO: Shall we consider holes for centroid computation? - for (int i =0; i < polygon.getPolygon().length() - 1; i++) { - centroidCalculator.addCoordinate(polygon.getPolygon().getX(i), polygon.getPolygon().getY(i)); - } - org.apache.lucene.geo.Polygon lucenePolygon = GeoShapeIndexer.toLucenePolygon(polygon); - addToExtent(lucenePolygon.minLon, lucenePolygon.maxLon, lucenePolygon.minLat, lucenePolygon.maxLat); - addTriangles(TriangleTreeLeaf.fromPolygon(coordinateEncoder, lucenePolygon)); - return null; - } - - @Override - public Void visit(MultiPolygon multiPolygon) { - for (Polygon polygon : multiPolygon) { - visit(polygon); - } - return null; - } - - @Override - public Void visit(Rectangle r) { - centroidCalculator.addCoordinate(r.getMinX(), r.getMinY()); - centroidCalculator.addCoordinate(r.getMaxX(), r.getMaxY()); - addToExtent(r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); - addTriangles(TriangleTreeLeaf.fromRectangle(coordinateEncoder, r)); - return null; + TriangleTreeNode[] nodes = new TriangleTreeNode[triangles.size()]; + for (int i = 0; i < triangles.size(); i++) { + nodes[i] = new TriangleTreeNode(triangles.get(i)); + addToExtent(nodes[i]); } + return createTree(nodes, 0, triangles.size() - 1, true); + } - @Override - public Void visit(Point point) { - centroidCalculator.addCoordinate(point.getX(), point.getY()); - addToExtent(point.getLon(), point.getLon(), point.getLat(), point.getLat()); - addTriangles(TriangleTreeLeaf.fromPoints(coordinateEncoder, point)); + /** Creates tree from sorted components (with range low and high inclusive) */ + private TriangleTreeNode createTree(TriangleTreeNode[] components, int low, int high, boolean splitX) { + if (low > high) { return null; } - - @Override - public Void visit(MultiPoint multiPoint) { - for (Point point : multiPoint) { - visit(point); + final int mid = (low + high) >>> 1; + if (low < high) { + Comparator comparator; + if (splitX) { + comparator = Comparator.comparingInt((TriangleTreeNode left) -> left.minX).thenComparingInt(left -> left.maxX); + } else { + comparator = Comparator.comparingInt((TriangleTreeNode left) -> left.minY).thenComparingInt(left -> left.maxY); } - return null; - } - - @Override - public Void visit(LinearRing ring) { - throw new IllegalArgumentException("invalid shape type found [LinearRing]"); - } - - @Override - public Void visit(Circle circle) { - throw new IllegalArgumentException("invalid shape type found [Circle]"); - } - - private void addToExtent(double minLon, double maxLon, double minLat, double maxLat) { - int minX = coordinateEncoder.encodeX(minLon); - int maxX = coordinateEncoder.encodeX(maxLon); - int minY = coordinateEncoder.encodeY(minLat); - int maxY = coordinateEncoder.encodeY(maxLat); - extent.addRectangle(minX, minY, maxX, maxY); + ArrayUtil.select(components, low, high + 1, mid, comparator); } + TriangleTreeNode newNode = components[mid]; + // find children + newNode.left = createTree(components, low, mid - 1, !splitX); + newNode.right = createTree(components, mid + 1, high, !splitX); - - public TriangleTreeNode build() { - if (triangles.size() == 1) { - return new TriangleTreeNode(triangles.get(0)); - } - TriangleTreeNode[] nodes = new TriangleTreeNode[triangles.size()]; - for (int i = 0; i < triangles.size(); i++) { - nodes[i] = new TriangleTreeNode(triangles.get(i)); - } - TriangleTreeNode root = createTree(nodes, 0, triangles.size() - 1, true); - return root; + // pull up max values to this node + if (newNode.left != null) { + newNode.maxX = Math.max(newNode.maxX, newNode.left.maxX); + newNode.maxY = Math.max(newNode.maxY, newNode.left.maxY); } - - /** Creates tree from sorted components (with range low and high inclusive) */ - private TriangleTreeNode createTree(TriangleTreeNode[] components, int low, int high, boolean splitX) { - if (low > high) { - return null; - } - final int mid = (low + high) >>> 1; - if (low < high) { - Comparator comparator; - if (splitX) { - comparator = (left, right) -> { - int ret = Double.compare(left.minX, right.minX); - if (ret == 0) { - ret = Double.compare(left.maxX, right.maxX); - } - return ret; - }; - } else { - comparator = (left, right) -> { - int ret = Double.compare(left.minY, right.minY); - if (ret == 0) { - ret = Double.compare(left.maxY, right.maxY); - } - return ret; - }; - } - ArrayUtil.select(components, low, high + 1, mid, comparator); - } - TriangleTreeNode newNode = components[mid]; - // find children - newNode.left = createTree(components, low, mid - 1, !splitX); - newNode.right = createTree(components, mid + 1, high, !splitX); - - // pull up max values to this node - if (newNode.left != null) { - newNode.maxX = Math.max(newNode.maxX, newNode.left.maxX); - newNode.maxY = Math.max(newNode.maxY, newNode.left.maxY); - } - if (newNode.right != null) { - newNode.maxX = Math.max(newNode.maxX, newNode.right.maxX); - newNode.maxY = Math.max(newNode.maxY, newNode.right.maxY); - } - return newNode; + if (newNode.right != null) { + newNode.maxX = Math.max(newNode.maxX, newNode.right.maxX); + newNode.maxY = Math.max(newNode.maxY, newNode.right.maxY); } + return newNode; } - /** - * Represents an inner node of the tree. - */ - static class TriangleTreeNode implements Writeable { + /** Represents an inner node of the tree. */ + private static class TriangleTreeNode { + /** type of component */ + public enum TYPE { + POINT, LINE, TRIANGLE + } /** minimum latitude of this geometry's bounding box area */ private int minY; /** maximum latitude of this geometry's bounding box area */ private int maxY; /** minimum longitude of this geometry's bounding box area */ private int minX; - /** maximum longitude of this geometry's bounding box area */ + /** maximum longitude of this geometry's bounding box area */ private int maxX; - // child components, or null. Note internal nodes might mot have - // a consistent bounding box. Internal nodes should not be accessed - // outside if this class. + // child components, or null. private TriangleTreeNode left; private TriangleTreeNode right; /** root node of edge tree */ - private TriangleTreeLeaf component; - - protected TriangleTreeNode(TriangleTreeLeaf component) { - this.minY = component.minY; - this.maxY = component.maxY; - this.minX = component.minX; - this.maxX = component.maxX; + private final ShapeField.DecodedTriangle component; + /** component type */ + private final TYPE type; + + private TriangleTreeNode(ShapeField.DecodedTriangle component) { + this.minY = Math.min(Math.min(component.aY, component.bY), component.cY); + this.maxY = Math.max(Math.max(component.aY, component.bY), component.cY); + this.minX = Math.min(Math.min(component.aX, component.bX), component.cX); + this.maxX = Math.max(Math.max(component.aX, component.bX), component.cX); this.component = component; + this.type = getType(component); + } + + private static TYPE getType(ShapeField.DecodedTriangle triangle) { + // the issue in lucene: https://github.com/apache/lucene-solr/pull/927 + // can help here + if (triangle.aX == triangle.bX && triangle.aY == triangle.bY) { + if (triangle.aX == triangle.cX && triangle.aY == triangle.cY) { + return TYPE.POINT; + } + return TYPE.LINE; + } else if ((triangle.aX == triangle.cX && triangle.aY == triangle.cY) || + (triangle.bX == triangle.cX && triangle.bY == triangle.cY)) { + return TYPE.LINE; + } else { + return TYPE.TRIANGLE; + } } - @Override - public void writeTo(StreamOutput out) throws IOException { - BytesStreamOutput scratchBuffer = new BytesStreamOutput(); + private void writeTo(ByteBuffersDataOutput out) throws IOException { + ByteBuffersDataOutput scratchBuffer = ByteBuffersDataOutput.newResettableInstance(); writeMetadata(out); writeComponent(out); if (left != null) { @@ -302,7 +168,8 @@ public void writeTo(StreamOutput out) throws IOException { } } - private void writeNode(StreamOutput out, int parentMaxX, int parentMaxY, BytesStreamOutput scratchBuffer) throws IOException { + private void writeNode(ByteBuffersDataOutput out, int parentMaxX, int parentMaxY, + ByteBuffersDataOutput scratchBuffer) throws IOException { out.writeVLong((long) parentMaxX - maxX); out.writeVLong((long) parentMaxY - maxY); int size = nodeSize(false, parentMaxX, parentMaxY, scratchBuffer); @@ -313,19 +180,19 @@ private void writeNode(StreamOutput out, int parentMaxX, int parentMaxY, BytesSt left.writeNode(out, maxX, maxY, scratchBuffer); } if (right != null) { - int rightSize = right.nodeSize(true, maxX, maxY,scratchBuffer); + int rightSize = right.nodeSize(true, maxX, maxY, scratchBuffer); out.writeVInt(rightSize); right.writeNode(out, maxX, maxY, scratchBuffer); } } - private void writeMetadata(StreamOutput out) throws IOException { + private void writeMetadata(ByteBuffersDataOutput out) { byte metadata = 0; metadata |= (left != null) ? (1 << 0) : 0; metadata |= (right != null) ? (1 << 1) : 0; - if (component.type == TriangleTreeLeaf.TYPE.POINT) { + if (type == TYPE.POINT) { metadata |= (1 << 2); - } else if (component.type == TriangleTreeLeaf.TYPE.LINE) { + } else if (type == TYPE.LINE) { metadata |= (1 << 3); } else { metadata |= (component.ab) ? (1 << 4) : 0; @@ -335,11 +202,11 @@ private void writeMetadata(StreamOutput out) throws IOException { out.writeByte(metadata); } - private void writeComponent(StreamOutput out) throws IOException { - if (component.type == TriangleTreeLeaf.TYPE.POINT) { + private void writeComponent(ByteBuffersDataOutput out) throws IOException { + if (type == TYPE.POINT) { out.writeVLong((long) maxX - component.aX); out.writeVLong((long) maxY - component.aY); - } else if (component.type == TriangleTreeLeaf.TYPE.LINE) { + } else if (type == TYPE.LINE) { out.writeVLong((long) maxX - component.aX); out.writeVLong((long) maxY - component.aY); out.writeVLong((long) maxX - component.bX); @@ -354,19 +221,19 @@ private void writeComponent(StreamOutput out) throws IOException { } } - public int nodeSize(boolean includeBox, int parentMaxX, int parentMaxY, BytesStreamOutput scratchBuffer) throws IOException { + private int nodeSize(boolean includeBox, int parentMaxX, int parentMaxY, ByteBuffersDataOutput scratchBuffer) throws IOException { int size =0; size++; //metadata size += componentSize(scratchBuffer); if (left != null) { - size += left.nodeSize(true, maxX, maxY, scratchBuffer); + size += left.nodeSize(true, maxX, maxY, scratchBuffer); } if (right != null) { int rightSize = right.nodeSize(true, maxX, maxY, scratchBuffer); scratchBuffer.reset(); scratchBuffer.writeVLong(rightSize); - size += scratchBuffer.size(); // jump size - size += rightSize; + size += scratchBuffer.size(); // jump size + size += rightSize; } if (includeBox) { int jumpSize = size; @@ -374,17 +241,17 @@ public int nodeSize(boolean includeBox, int parentMaxX, int parentMaxY, BytesStr scratchBuffer.writeVLong((long) parentMaxX - maxX); scratchBuffer.writeVLong((long) parentMaxY - maxY); scratchBuffer.writeVLong(jumpSize); - size += scratchBuffer.size();// box + size += scratchBuffer.size(); // box size } return size; } - public int componentSize(BytesStreamOutput scratchBuffer) throws IOException { + private int componentSize(ByteBuffersDataOutput scratchBuffer) throws IOException { scratchBuffer.reset(); - if (component.type == TriangleTreeLeaf.TYPE.POINT) { + if (type == TYPE.POINT) { scratchBuffer.writeVLong((long) maxX - component.aX); scratchBuffer.writeVLong((long) maxY - component.aY); - } else if (component.type == TriangleTreeLeaf.TYPE.LINE) { + } else if (type == TYPE.LINE) { scratchBuffer.writeVLong((long) maxX - component.aX); scratchBuffer.writeVLong((long) maxY - component.aY); scratchBuffer.writeVLong((long) maxX - component.bX); @@ -397,236 +264,7 @@ public int componentSize(BytesStreamOutput scratchBuffer) throws IOException { scratchBuffer.writeVLong((long) maxX - component.cX); scratchBuffer.writeVLong((long) maxY - component.cY); } - return scratchBuffer.size(); - } - - } - - /** - * Represents an leaf of the tree containing one of the triangles. - */ - static class TriangleTreeLeaf { - - public enum TYPE { - POINT, LINE, TRIANGLE - } - - int minX, maxX, minY, maxY; - int aX, aY, bX, bY, cX, cY; - boolean ab, bc, ca; - TYPE type; - - // constructor for points - TriangleTreeLeaf(int aXencoded, int aYencoded) { - encodePoint(aXencoded, aYencoded); - } - - // constructor for points and lines - TriangleTreeLeaf(int aXencoded, int aYencoded, int bXencoded, int bYencoded) { - if (aXencoded == bXencoded && aYencoded == bYencoded) { - encodePoint(aXencoded, aYencoded); - } else { - encodeLine(aXencoded, aYencoded, bXencoded, bYencoded); - } - } - - // generic constructor - TriangleTreeLeaf(int aXencoded, int aYencoded, boolean ab, - int bXencoded, int bYencoded, boolean bc, - int cXencoded, int cYencoded, boolean ca) { - if (aXencoded == bXencoded && aYencoded == bYencoded) { - if (aXencoded == cXencoded && aYencoded == cYencoded) { - encodePoint(aYencoded, aXencoded); - } else { - encodeLine(aYencoded, aXencoded, cYencoded, cXencoded); - return; - } - } else if (aXencoded == cXencoded && aYencoded == cYencoded) { - encodeLine(aYencoded, aXencoded, bYencoded, bXencoded); - } else { - encodeTriangle(aXencoded, aYencoded, ab, bXencoded, bYencoded, bc, cXencoded, cYencoded, ca); - } - } - - private void encodePoint(int aXencoded, int aYencoded) { - this.type = TYPE.POINT; - aX = aXencoded; - aY = aYencoded; - minX = aX; - maxX = aX; - minY = aY; - maxY = aY; - } - - private void encodeLine(int aXencoded, int aYencoded, int bXencoded, int bYencoded) { - this.type = TYPE.LINE; - //rotate edges and place minX at the beginning - if (aXencoded > bXencoded) { - aX = bXencoded; - aY = bYencoded; - bX = aXencoded; - bY = aYencoded; - } else { - aX = aXencoded; - aY = aYencoded; - bX = bXencoded; - bY = bYencoded; - } - this.minX = aX; - this.maxX = bX; - this.minY = Math.min(aY, bY); - this.maxY = Math.max(aY, bY); - } - - private void encodeTriangle(int aXencoded, int aYencoded, boolean abFromShape, - int bXencoded, int bYencoded, boolean bcFromShape, - int cXencoded, int cYencoded, boolean caFromShape) { - - int aX, aY, bX, bY, cX, cY; - boolean ab, bc, ca; - //change orientation if CW - if (GeoUtils.orient(aXencoded, aYencoded, bXencoded, bYencoded, cXencoded, cYencoded) == -1) { - aX = cXencoded; - bX = bXencoded; - cX = aXencoded; - aY = cYencoded; - bY = bYencoded; - cY = aYencoded; - ab = bcFromShape; - bc = abFromShape; - ca = caFromShape; - } else { - aX = aXencoded; - bX = bXencoded; - cX = cXencoded; - aY = aYencoded; - bY = bYencoded; - cY = cYencoded; - ab = abFromShape; - bc = bcFromShape; - ca = caFromShape; - } - //rotate edges and place minX at the beginning - if (bX < aX || cX < aX) { - if (bX < cX) { - int tempX = aX; - int tempY = aY; - boolean tempBool = ab; - aX = bX; - aY = bY; - ab = bc; - bX = cX; - bY = cY; - bc = ca; - cX = tempX; - cY = tempY; - ca = tempBool; - } else if (cX < aX) { - int tempX = aX; - int tempY = aY; - boolean tempBool = ab; - aX = cX; - aY = cY; - ab = ca; - cX = bX; - cY = bY; - ca = bc; - bX = tempX; - bY = tempY; - bc = tempBool; - } - } else if (aX == bX && aX == cX) { - //degenerated case, all points with same longitude - //we need to prevent that aX is in the middle (not part of the MBS) - if (bY < aY || cY < aY) { - if (bY < cY) { - int tempX = aX; - int tempY = aY; - boolean tempBool = ab; - aX = bX; - aY = bY; - ab = bc; - bX = cX; - bY = cY; - bc = ca; - cX = tempX; - cY = tempY; - ca = tempBool; - } else if (cY < aY) { - int tempX = aX; - int tempY = aY; - boolean tempBool = ab; - aX = cX; - aY = cY; - ab = ca; - cX = bX; - cY = bY; - ca = bc; - bX = tempX; - bY = tempY; - bc = tempBool; - } - } - } - this.aX = aX; - this.aY = aY; - this.bX = bX; - this.bY = bY; - this.cX = cX; - this.cY = cY; - this.ab = ab; - this.bc = bc; - this.ca = ca; - this.minX = aX; - this.maxX = Math.max(aX, Math.max(bX, cX)); - this.minY = Math.min(aY, Math.min(bY, cY)); - this.maxY = Math.max(aY, Math.max(bY, cY)); - type = TYPE.TRIANGLE; - } - - private static List fromPoints(CoordinateEncoder encoder, Point... points) { - List triangles = new ArrayList<>(points.length); - for (int i = 0; i < points.length; i++) { - triangles.add(new TriangleTreeLeaf(encoder.encodeX(points[i].getX()), encoder.encodeY(points[i].getY()))); - } - return triangles; - } - - private static List fromRectangle(CoordinateEncoder encoder, Rectangle... rectangles) { - List triangles = new ArrayList<>(2 * rectangles.length); - for (Rectangle r : rectangles) { - triangles.add(new TriangleTreeLeaf( - encoder.encodeX(r.getMinX()), encoder.encodeY(r.getMinY()), true, - encoder.encodeX(r.getMaxX()), encoder.encodeY(r.getMinY()), false, - encoder.encodeX(r.getMinX()), encoder.encodeY(r.getMaxY()), true)); - triangles.add(new TriangleTreeLeaf( - encoder.encodeX(r.getMinX()), encoder.encodeY(r.getMaxY()), false, - encoder.encodeX(r.getMaxX()), encoder.encodeY(r.getMinY()), true, - encoder.encodeX(r.getMaxX()), encoder.encodeY(r.getMaxY()), true)); - } - return triangles; - } - - private static List fromLine(CoordinateEncoder encoder, org.apache.lucene.geo.Line line) { - List triangles = new ArrayList<>(line.numPoints() - 1); - for (int i = 0, j = 1; i < line.numPoints() - 1; i++, j++) { - triangles.add(new TriangleTreeLeaf(encoder.encodeX(line.getLon(i)), encoder.encodeY(line.getLat(i)), - encoder.encodeX(line.getLon(j)), encoder.encodeY(line.getLat(j)))); - } - return triangles; - } - - private static List fromPolygon(CoordinateEncoder encoder, org.apache.lucene.geo.Polygon polygon) { - // TODO: We are going to be tessellating the polygon twice, can we do something? - // TODO: Tessellator seems to have some reference to the encoding but does not need to have. - List tessellation = Tessellator.tessellate(polygon); - List triangles = new ArrayList<>(tessellation.size()); - for (Tessellator.Triangle t : tessellation) { - triangles.add(new TriangleTreeLeaf(encoder.encodeX(t.getX(0)), encoder.encodeY(t.getY(0)), t.isEdgefromPolygon(0), - encoder.encodeX(t.getX(1)), encoder.encodeY(t.getY(1)), t.isEdgefromPolygon(1), - encoder.encodeX(t.getX(2)), encoder.encodeY(t.getY(2)), t.isEdgefromPolygon(2))); - } - return triangles; + return Math.toIntExact(scratchBuffer.size()); } } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 893d020ebc657..169597f5d9572 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -18,7 +18,12 @@ */ package org.elasticsearch.index.fielddata; +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.spatial.util.GeoRelationUtils; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.CoordinateEncoder; import org.elasticsearch.common.geo.Extent; import org.elasticsearch.common.geo.GeoPoint; @@ -26,15 +31,17 @@ import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.TriangleTreeReader; import org.elasticsearch.common.geo.TriangleTreeWriter; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.GeographyValidator; import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.search.aggregations.support.ValuesSourceType; import java.io.IOException; import java.text.ParseException; +import java.util.Arrays; +import java.util.List; /** * A stateful lightweight per document set of geo values. @@ -84,8 +91,10 @@ protected MultiGeoValues() { public static class GeoPointValue implements GeoValue { private final GeoPoint geoPoint; + private final BoundingBox boundingBox; public GeoPointValue(GeoPoint geoPoint) { + this.boundingBox = new BoundingBox(); this.geoPoint = geoPoint; } @@ -95,13 +104,14 @@ public GeoPoint geoPoint() { @Override public BoundingBox boundingBox() { - return new BoundingBox(geoPoint); + boundingBox.reset(geoPoint); + return boundingBox; } @Override public GeoRelation relate(Rectangle rectangle) { if (GeoRelationUtils.pointInRectPrecise(geoPoint.lat(), geoPoint.lon(), - rectangle.getMinLat(), rectangle.getMaxLat(), rectangle.getMinLon(), rectangle.getMaxLon())) { + rectangle.getMinLat(), rectangle.getMaxLat(), rectangle.getMinLon(), rectangle.getMaxLon())) { return GeoRelation.QUERY_CROSSES; } return GeoRelation.QUERY_DISJOINT; @@ -127,18 +137,17 @@ public static class GeoShapeValue implements GeoValue { private static final WellKnownText MISSING_GEOMETRY_PARSER = new WellKnownText(true, new GeographyValidator(true)); private final TriangleTreeReader reader; + private final BoundingBox boundingBox; public GeoShapeValue(TriangleTreeReader reader) { this.reader = reader; + this.boundingBox = new BoundingBox(); } @Override public BoundingBox boundingBox() { - try { - return new BoundingBox(reader.getExtent(), GeoShapeCoordinateEncoder.INSTANCE); - } catch (IOException e) { - throw new IllegalStateException("unable to read bounding box", e); - } + boundingBox.reset(reader.getExtent(), GeoShapeCoordinateEncoder.INSTANCE); + return boundingBox; } /** @@ -150,20 +159,12 @@ public GeoRelation relate(Rectangle rectangle) { int maxX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMaxX()); int minY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMinY()); int maxY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMaxY()); - try { - return reader.relate(minX, minY, maxX, maxY); - } catch (IOException e) { - throw new IllegalStateException("unable to check intersection", e); - } + return reader.relate(minX, minY, maxX, maxY); } @Override public double lat() { - try { - return reader.getCentroidY(); - } catch (IOException e) { - throw new IllegalStateException("unable to read centroid of shape", e); - } + return reader.getCentroidY(); } /** @@ -171,26 +172,40 @@ public double lat() { */ @Override public double lon() { - try { - return reader.getCentroidX(); - } catch (IOException e) { - throw new IllegalStateException("unable to read centroid of shape", e); - } + return reader.getCentroidX(); } public static GeoShapeValue missing(String missing) { try { Geometry geometry = MISSING_GEOMETRY_PARSER.fromWKT(missing); - TriangleTreeWriter writer = new TriangleTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); + ShapeField.DecodedTriangle[] triangles = toDecodedTriangles(geometry); + TriangleTreeWriter writer = + new TriangleTreeWriter(Arrays.asList(triangles), GeoShapeCoordinateEncoder.INSTANCE, + new CentroidCalculator(geometry)); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); writer.writeTo(output); - TriangleTreeReader reader = new TriangleTreeReader(GeoShapeCoordinateEncoder.INSTANCE); - reader.reset(output.bytes().toBytesRef()); + TriangleTreeReader reader = new TriangleTreeReader(GeoShapeCoordinateEncoder.INSTANCE); + reader.reset(new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size()))); return new GeoShapeValue(reader); } catch (IOException | ParseException e) { throw new IllegalArgumentException("Can't apply missing value [" + missing + "]", e); } } + + private static ShapeField.DecodedTriangle[] toDecodedTriangles(Geometry geometry) { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + List fields = indexer.indexShape(null, geometry); + ShapeField.DecodedTriangle[] triangles = new ShapeField.DecodedTriangle[fields.size()]; + final byte[] scratch = new byte[7 * Integer.BYTES]; + for (int i = 0; i < fields.size(); i++) { + BytesRef bytesRef = fields.get(i).binaryValue(); + assert bytesRef.length == 7 * Integer.BYTES; + System.arraycopy(bytesRef.bytes, bytesRef.offset, scratch, 0, 7 * Integer.BYTES); + ShapeField.decodeTriangle(scratch, triangles[i] = new ShapeField.DecodedTriangle()); + } + return triangles; + } } /** @@ -205,14 +220,17 @@ public interface GeoValue { } public static class BoundingBox { - public final double top; - public final double bottom; - public final double negLeft; - public final double negRight; - public final double posLeft; - public final double posRight; - - public BoundingBox(Extent extent, CoordinateEncoder coordinateEncoder) { + public double top; + public double bottom; + public double negLeft; + public double negRight; + public double posLeft; + public double posRight; + + private BoundingBox() { + } + + private void reset(Extent extent, CoordinateEncoder coordinateEncoder) { this.top = coordinateEncoder.decodeY(extent.top); this.bottom = coordinateEncoder.decodeY(extent.bottom); if (extent.negLeft == Integer.MAX_VALUE) { @@ -237,7 +255,7 @@ public BoundingBox(Extent extent, CoordinateEncoder coordinateEncoder) { } } - BoundingBox(GeoPoint point) { + private void reset(GeoPoint point) { this.top = point.lat(); this.bottom = point.lat(); if (point.lon() < 0) { @@ -252,6 +270,7 @@ public BoundingBox(Extent extent, CoordinateEncoder coordinateEncoder) { this.posRight = point.lon(); } } + /** * @return the minimum y-coordinate of the extent */ @@ -280,6 +299,5 @@ public double maxX() { return Math.max(negRight, posRight); } - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index e9c6088205b92..bd3a57752115a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -18,14 +18,17 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.ShapeField; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.geo.builders.ShapeBuilder; @@ -81,7 +84,8 @@ public interface Indexer { List indexShape(ParseContext context, Processed shape); - void indexDocValueField(ParseContext context, Processed shape); + void indexDocValueField(ParseContext context, ShapeField.DecodedTriangle[] triangles, + CentroidCalculator centroidCalculator); } /** @@ -436,10 +440,18 @@ public void parse(ParseContext context) throws IOException { shape = geometryIndexer.prepareForIndexing(geometry); } - List fields = new ArrayList<>(); - fields.addAll(geometryIndexer.indexShape(context, shape)); + List fields = new ArrayList<>(geometryIndexer.indexShape(context, shape)); + final byte[] scratch = new byte[7 * Integer.BYTES]; if (fieldType().hasDocValues()) { - geometryIndexer.indexDocValueField(context, shape); + // doc values are generated from the indexed fields. + ShapeField.DecodedTriangle[] triangles = new ShapeField.DecodedTriangle[fields.size()]; + for (int i =0; i < fields.size(); i++) { + BytesRef bytesRef = fields.get(i).binaryValue(); + assert bytesRef.length == 7 * Integer.BYTES; + System.arraycopy(bytesRef.bytes, bytesRef.offset, scratch, 0, 7 * Integer.BYTES); + ShapeField.decodeTriangle(scratch, triangles[i] = new ShapeField.DecodedTriangle()); + } + geometryIndexer.indexDocValueField(context, triangles, new CentroidCalculator((Geometry) shape)); } createFieldNamesField(context, fields); for (IndexableField field : fields) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java index 9702fcb2403e3..10884ba41ee41 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryGeoShapeDocValuesField.java @@ -18,45 +18,43 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.TriangleTreeWriter; -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.GeometryCollection; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class BinaryGeoShapeDocValuesField extends CustomDocValuesField { - private List geometries; + private final List triangles; + private final CentroidCalculator centroidCalculator; - public BinaryGeoShapeDocValuesField(String name, Geometry geometry) { + public BinaryGeoShapeDocValuesField(String name, ShapeField.DecodedTriangle[] triangles, CentroidCalculator centroidCalculator) { super(name); - this.geometries = new ArrayList<>(1); - add(geometry); + this.triangles = new ArrayList<>(triangles.length); + this.centroidCalculator = centroidCalculator; + this.triangles.addAll(Arrays.asList(triangles)); } - public void add(Geometry geometry) { - geometries.add(geometry); + public void add(ShapeField.DecodedTriangle[] triangles, CentroidCalculator centroidCalculator) { + this.triangles.addAll(Arrays.asList(triangles)); + this.centroidCalculator.addFrom(centroidCalculator); } @Override public BytesRef binaryValue() { try { - final Geometry geometry; - if (geometries.size() > 1) { - geometry = new GeometryCollection(geometries); - } else { - geometry = geometries.get(0); - } - final TriangleTreeWriter writer = new TriangleTreeWriter(geometry, GeoShapeCoordinateEncoder.INSTANCE); - BytesStreamOutput output = new BytesStreamOutput(); + final TriangleTreeWriter writer = new TriangleTreeWriter(triangles, GeoShapeCoordinateEncoder.INSTANCE, centroidCalculator); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); writer.writeTo(output); - return output.bytes().toBytesRef(); + return new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size())); } catch (IOException e) { throw new ElasticsearchException("failed to encode shape", e); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java index 4b8db174ca7df..9c8cea248a049 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeIndexer.java @@ -21,8 +21,10 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.document.ShapeField; import org.apache.lucene.index.IndexableField; import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; @@ -197,14 +199,14 @@ public List indexShape(ParseContext context, Geometry shape) { } @Override - public void indexDocValueField(ParseContext context, Geometry shape) { + public void indexDocValueField(ParseContext context, ShapeField.DecodedTriangle[] triangles, CentroidCalculator calculator) { BinaryGeoShapeDocValuesField docValuesField = (BinaryGeoShapeDocValuesField) context.doc().getByKey(name); if (docValuesField == null) { - docValuesField = new BinaryGeoShapeDocValuesField(name, shape); + docValuesField = new BinaryGeoShapeDocValuesField(name, triangles, calculator); context.doc().addWithKey(name, docValuesField); } else { - docValuesField.add(shape); + docValuesField.add(triangles, calculator); } } @@ -1075,7 +1077,6 @@ private void addFields(IndexableField[] fields) { } } - public static org.apache.lucene.geo.Polygon toLucenePolygon(Polygon polygon) { org.apache.lucene.geo.Polygon[] holes = new org.apache.lucene.geo.Polygon[polygon.getNumberOfHoles()]; for(int i = 0; i indexShape(ParseContext context, Shape shape) { } @Override - public void indexDocValueField(ParseContext context, Shape shape) { + public void indexDocValueField(ParseContext context, ShapeField.DecodedTriangle[] triangles, CentroidCalculator centroidCalculator) { throw new UnsupportedOperationException("doc values not supported for legacy shape indexer"); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java index 43c5e4a02134b..c3933b9361310 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java @@ -18,6 +18,9 @@ */ package org.elasticsearch.common.geo; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.Point; import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.equalTo; @@ -25,20 +28,28 @@ public class CentroidCalculatorTests extends ESTestCase { public void test() { - double[] x = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; - double[] y = new double[] { 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 }; - double[] xRunningAvg = new double[] { 1, 1.5, 2.0, 2.5, 3, 3.5, 4, 4.5, 5, 5.5 }; - double[] yRunningAvg = new double[] { 10, 15, 20, 25, 30, 35, 40, 45, 50, 55 }; - CentroidCalculator calculator = new CentroidCalculator(); - for (int i = 0; i < 10; i++) { - calculator.addCoordinate(x[i], y[i]); + double[] y = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; + double[] x = new double[] { 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 }; + double[] yRunningAvg = new double[] { 1, 1.5, 2.0, 2.5, 3, 3.5, 4, 4.5, 5, 5.5 }; + double[] xRunningAvg = new double[] { 10, 15, 20, 25, 30, 35, 40, 45, 50, 55 }; + + Point point = new Point(x[0], y[0]); + CentroidCalculator calculator = new CentroidCalculator(point); + assertThat(calculator.getX(), equalTo(xRunningAvg[0])); + assertThat(calculator.getY(), equalTo(yRunningAvg[0])); + for (int i = 1; i < 10; i++) { + double[] subX = new double[i + 1]; + double[] subY = new double[i + 1]; + System.arraycopy(x, 0, subX, 0, i + 1); + System.arraycopy(y, 0, subY, 0, i + 1); + Geometry geometry = new Line(subX, subY); + calculator = new CentroidCalculator(geometry); assertThat(calculator.getX(), equalTo(xRunningAvg[i])); assertThat(calculator.getY(), equalTo(yRunningAvg[i])); } - CentroidCalculator otherCalculator = new CentroidCalculator(); - otherCalculator.addCoordinate(0.0, 0.0); + CentroidCalculator otherCalculator = new CentroidCalculator(new Point(0, 0)); calculator.addFrom(otherCalculator); - assertThat(calculator.getX(), equalTo(5.0)); - assertThat(calculator.getY(), equalTo(50.0)); + assertThat(calculator.getX(), equalTo(50.0)); + assertThat(calculator.getY(), equalTo(5.0)); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java index 378fe9e3f054f..db230df45a133 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java @@ -18,15 +18,11 @@ */ package org.elasticsearch.common.geo; -import org.elasticsearch.Version; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.test.AbstractWireSerializingTestCase; - -import java.io.IOException; +import org.elasticsearch.test.ESTestCase; import static org.hamcrest.Matchers.equalTo; -public class ExtentTests extends AbstractWireSerializingTestCase { +public class ExtentTests extends ESTestCase { public void testFromPoint() { int x = randomFrom(-1, 0, 1); @@ -89,21 +85,4 @@ public void testAddRectangle() { assertThat(extent.minY(), equalTo(bottomLeftY2)); assertThat(extent.maxY(), equalTo(topRightY2)); } - - @Override - protected Extent createTestInstance() { - return new Extent(randomIntBetween(-10, 10), randomIntBetween(-10, 10), randomIntBetween(-10, 10), - randomIntBetween(-10, 10), randomIntBetween(-10, 10), randomIntBetween(-10, 10)); - } - - @Override - protected Writeable.Reader instanceReader() { - return Extent::new; - } - - @Override - protected Object copyInstance(Object instance, Version version) throws IOException { - Extent other = (Extent) instance; - return new Extent(other.top, other.bottom, other.negLeft, other.negRight, other.posLeft, other.posRight); - } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java index eb54ecba06ae4..296029464ca7d 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -18,16 +18,23 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.document.ShapeField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.index.mapper.GeoShapeIndexer; + import java.io.IOException; +import java.util.Arrays; +import java.util.List; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; @@ -39,13 +46,28 @@ public static void assertRelation(GeoRelation expectedRelation, TriangleTreeRead assertThat(actualRelation, equalTo(expectedRelation)); } + public static ShapeField.DecodedTriangle[] toDecodedTriangles(Geometry geometry) throws IOException { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + List fields = indexer.indexShape(null, geometry); + ShapeField.DecodedTriangle[] triangles = new ShapeField.DecodedTriangle[fields.size()]; + final byte[] scratch = new byte[7 * Integer.BYTES]; + for (int i = 0; i < fields.size(); i++) { + BytesRef bytesRef = fields.get(i).binaryValue(); + assert bytesRef.length == 7 * Integer.BYTES; + System.arraycopy(bytesRef.bytes, bytesRef.offset, scratch, 0, 7 * Integer.BYTES); + ShapeField.decodeTriangle(scratch, triangles[i] = new ShapeField.DecodedTriangle()); + } + return triangles; + } + public static TriangleTreeReader triangleTreeReader(Geometry geometry, CoordinateEncoder encoder) throws IOException { - TriangleTreeWriter writer = new TriangleTreeWriter(geometry, encoder); - BytesStreamOutput output = new BytesStreamOutput(); + ShapeField.DecodedTriangle[] triangles = toDecodedTriangles(geometry); + TriangleTreeWriter writer = new TriangleTreeWriter(Arrays.asList(triangles), encoder, new CentroidCalculator(geometry)); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); writer.writeTo(output); - output.close(); TriangleTreeReader reader = new TriangleTreeReader(encoder); - reader.reset(output.bytes().toBytesRef()); + reader.reset(new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size()))); return reader; } diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java index 99c24fe1b6298..049ae31d06f47 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -48,64 +48,69 @@ public class TriangleTreeTests extends ESTestCase { public void testRectangleShape() throws IOException { for (int i = 0; i < 1000; i++) { - int minX = randomIntBetween(-80, 70); - int maxX = randomIntBetween(minX + 10, 80); - int minY = randomIntBetween(-80, 70); - int maxY = randomIntBetween(minY + 10, 80); + int minX = randomIntBetween(-40, -1); + int maxX = randomIntBetween(1, 40); + int minY = randomIntBetween(-40, -1); + int maxY = randomIntBetween(1, 40); double[] x = new double[]{minX, maxX, maxX, minX, minX}; double[] y = new double[]{minY, minY, maxY, maxY, minY}; Geometry rectangle = randomBoolean() ? new Polygon(new LinearRing(x, y), Collections.emptyList()) : new Rectangle(minX, maxX, maxY, minY); - TriangleTreeReader reader = triangleTreeReader(rectangle, TestCoordinateEncoder.INSTANCE); + TriangleTreeReader reader = triangleTreeReader(rectangle, GeoShapeCoordinateEncoder.INSTANCE); - assertThat(Extent.fromPoints(minX, minY, maxX, maxY), equalTo(reader.getExtent())); - // encoder loses precision when casting to integer, so centroid is calculated using integer division here - assertThat(reader.getCentroidX(), equalTo((double) ((minX + maxX) / 2))); - assertThat(reader.getCentroidY(), equalTo((double) ((minY + maxY) / 2))); + Extent expectedExtent = getExtentFromBox(minX, minY, maxX, maxY); + assertThat(expectedExtent, equalTo(reader.getExtent())); + // centroid is calculated using original double values but then loses precision as it is serialized as an integer + int encodedCentroidX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(((double) minX + maxX) / 2); + int encodedCentroidY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(((double) minY + maxY) / 2); + assertThat(reader.getCentroidX(), equalTo(GeoShapeCoordinateEncoder.INSTANCE.decodeX(encodedCentroidX))); + assertThat(reader.getCentroidY(), equalTo(GeoShapeCoordinateEncoder.INSTANCE.decodeY(encodedCentroidY))); // box-query touches bottom-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), - minY - randomIntBetween(1, 180), minX, minY)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), + minY - randomIntBetween(1, 90 + minY), minX, minY)); // box-query touches bottom-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, minY - randomIntBetween(1, 180), - maxX + randomIntBetween(1, 180), minY)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(maxX, minY - randomIntBetween(1, 90 + minY), + maxX + randomIntBetween(1, 180 - maxX), minY)); // box-query touches top-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(maxX, maxY, maxX + randomIntBetween(1, 180), - maxY + randomIntBetween(1, 180))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(maxX, maxY, maxX + randomIntBetween(1, 180 - maxX), + maxY + randomIntBetween(1, 90 - maxY))); // box-query touches top-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), maxY, minX, - maxY + randomIntBetween(1, 180))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), maxY, minX, + maxY + randomIntBetween(1, 90 - maxY))); // box-query fully-enclosed inside rectangle - assertRelation(GeoRelation.QUERY_INSIDE,reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); + assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(3 * (minX + maxX) / 4, 3 * (minY + maxY) / 4, + 3 * (maxX + minX) / 4, 3 * (maxY + minY) / 4)); // box-query fully-contains poly - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 180), - minY - randomIntBetween(1, 180), maxX + randomIntBetween(1, 180), maxY + randomIntBetween(1, 180))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), + minY - randomIntBetween(1, 90 + minY), maxX + randomIntBetween(1, 180 - maxX), + maxY + randomIntBetween(1, 90 - maxY))); // box-query half-in-half-out-right - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(3 * (minX + maxX) / 4, 3 * (minY + maxY) / 4, + maxX + randomIntBetween(1, 90 - maxY), 3 * (maxY + minY) / 4)); // box-query half-in-half-out-left - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(minX - randomIntBetween(1, 1000), (3 * minY + maxY) / 4, - (3 * maxX + minX) / 4, (3 * maxY + minY) / 4)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), + 3 * (minY + maxY) / 4, 3 * (maxX + minX) / 4, 3 * (maxY + minY) / 4)); // box-query half-in-half-out-top - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, (3 * minY + maxY) / 4, - maxX + randomIntBetween(1, 1000), maxY + randomIntBetween(1, 1000))); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(3 * (minX + maxX) / 4, 3 * (minY + maxY) / 4, + maxX + randomIntBetween(1, 180 - maxX), maxY + randomIntBetween(1, 90 - maxY))); // box-query half-in-half-out-bottom - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints((3 * minX + maxX) / 4, minY - randomIntBetween(1, 1000), - maxX + randomIntBetween(1, 1000), (3 * maxY + minY) / 4)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(3 * (minX + maxX) / 4, + minY - randomIntBetween(1, 90 + minY), maxX + randomIntBetween(1, 180 - maxX), + 3 * (maxY + minY) / 4)); // box-query outside to the right - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX + randomIntBetween(1, 1000), minY, - maxX + randomIntBetween(1001, 2000), maxY)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(maxX + randomIntBetween(1, 180 - maxX), minY, + maxX + randomIntBetween(1, 180 - maxX), maxY)); // box-query outside to the left - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(maxX - randomIntBetween(1001, 2000), minY, - minX - randomIntBetween(1, 1000), maxY)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(maxX - randomIntBetween(1, 180 - maxX), minY, + minX - randomIntBetween(1, 180 + minX), maxY)); // box-query outside to the top - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, maxY + randomIntBetween(1, 1000), maxX, - maxY + randomIntBetween(1001, 2000))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(minX, maxY + randomIntBetween(1, 90 - maxY), maxX, + maxY + randomIntBetween(1, 90 - maxY))); // box-query outside to the bottom - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(minX, minY - randomIntBetween(1001, 2000), maxX, - minY - randomIntBetween(1, 1000))); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(minX, minY - randomIntBetween(1, 90 + minY), maxX, + minY - randomIntBetween(1, 90 + minY))); } } @@ -117,10 +122,10 @@ public void testPacManPolygon() throws Exception { // test cell crossing poly TriangleTreeReader reader = triangleTreeReader(new Polygon(new LinearRing(py, px), Collections.emptyList()), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(-5, -6, 2, -2)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(-5, -6, 2, -2)); } // adapted from org.apache.lucene.geo.TestPolygon2D#testMultiPolygon @@ -128,14 +133,14 @@ public void testPolygonWithHole() throws Exception { Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), Collections.singletonList(new LinearRing(new double[]{-10, 10, 10, -10, -10}, new double[]{-10, -10, 10, 10, -10}))); - TriangleTreeReader reader = triangleTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); + TriangleTreeReader reader = triangleTreeReader(polyWithHole, GeoShapeCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(6, -6, 6, -6)); // in the hole - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(25, -25, 25, -25)); // on the mainland - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(51, 51, 52, 52)); // outside of mainland - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-60, -60, 60, 60)); // enclosing us completely - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(49, 49, 51, 51)); // overlapping the mainland - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(9, 9, 11, 11)); // overlapping the hole + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(6, -6, 6, -6)); // in the hole + assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(25, -25, 25, -25)); // on the mainland + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(51, 51, 52, 52)); // outside of mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-60, -60, 60, 60)); // enclosing us completely + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(49, 49, 51, 51)); // overlapping the mainland + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(9, 9, 11, 11)); // overlapping the hole } public void testCombPolygon() throws Exception { @@ -146,11 +151,11 @@ public void testCombPolygon() throws Exception { double[] hy = {1, 20, 20, 1, 1}; Polygon polyWithHole = new Polygon(new LinearRing(px, py), Collections.singletonList(new LinearRing(hx, hy))); - TriangleTreeReader reader = triangleTreeReader(polyWithHole, TestCoordinateEncoder.INSTANCE); + TriangleTreeReader reader = triangleTreeReader(polyWithHole, GeoShapeCoordinateEncoder.INSTANCE); // test cell crossing poly - assertRelation(GeoRelation.QUERY_INSIDE, reader, Extent.fromPoints(5, 10, 5, 10)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(15, 10, 15, 10)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(25, 10, 25, 10)); + assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(5, 10, 5, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(15, 10, 15, 10)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(25, 10, 25, 10)); } public void testPacManClosedLineString() throws Exception { @@ -159,11 +164,11 @@ public void testPacManClosedLineString() throws Exception { double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; // test cell crossing poly - TriangleTreeReader reader = triangleTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); + TriangleTreeReader reader = triangleTreeReader(new Line(px, py), GeoShapeCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(-5, -6, 2, -2)); } public void testPacManLineString() throws Exception { @@ -172,11 +177,11 @@ public void testPacManLineString() throws Exception { double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5}; // test cell crossing poly - TriangleTreeReader reader = triangleTreeReader(new Line(px, py), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(2, -1, 11, 1)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-12, -12, 12, 12)); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(-2, -1, 2, 0)); - assertRelation(GeoRelation.QUERY_DISJOINT, reader, Extent.fromPoints(-5, -6, 2, -2)); + TriangleTreeReader reader = triangleTreeReader(new Line(px, py), GeoShapeCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12)); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0)); + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(-5, -6, 2, -2)); } public void testPacManPoints() throws Exception { @@ -202,8 +207,8 @@ public void testPacManPoints() throws Exception { int yMax = 9; // test cell crossing poly - TriangleTreeReader reader = triangleTreeReader(new MultiPoint(points), TestCoordinateEncoder.INSTANCE); - assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(xMin, yMin, xMax, yMax)); + TriangleTreeReader reader = triangleTreeReader(new MultiPoint(points), GeoShapeCoordinateEncoder.INSTANCE); + assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(xMin, yMin, xMax, yMax)); } public void testRandomMultiLineIntersections() throws IOException { @@ -271,6 +276,14 @@ private Extent bufferedExtentFromGeoPoint(double x, double y, double extentSize) return Extent.fromPoints(xMin, yMin, xMax, yMax); } + private static Extent getExtentFromBox(double bottomLeftX, double bottomLeftY, double topRightX, double topRightY) { + return Extent.fromPoints(GeoShapeCoordinateEncoder.INSTANCE.encodeX(bottomLeftX), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(bottomLeftY), + GeoShapeCoordinateEncoder.INSTANCE.encodeX(topRightX), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(topRightY)); + + } + private boolean intersects(Geometry g, Point p, double extentSize) throws IOException { Extent bufferBounds = bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 13b9ee51d8ae3..41655905e4b04 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -29,6 +29,9 @@ import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.geo.CentroidCalculator; +import org.elasticsearch.common.geo.GeoTestUtils; +import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; @@ -90,7 +93,9 @@ public void testFieldMissing() throws IOException { }, new GeoPointFieldMapper.GeoPointFieldType()); testCase(new MatchAllDocsQuery(), "wrong_field", randomPrecision(), iw -> { - iw.addDocument(Collections.singleton(new BinaryGeoShapeDocValuesField(FIELD_NAME, new Point(10D, 10D)))); + iw.addDocument(Collections.singleton( + new BinaryGeoShapeDocValuesField(FIELD_NAME, GeoTestUtils.toDecodedTriangles(new Point(10D, 10D)), + new CentroidCalculator(new Point(10D, 10D))))); }, geoGrid -> { assertEquals(0, geoGrid.getBuckets().size()); }, new GeoShapeFieldMapper.GeoShapeFieldType()); @@ -165,7 +170,9 @@ public void testGeoShapeWithSeveralDocs() throws IOException { } distinctHashesPerDoc.add(hash); if (usually()) { - document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, new MultiPoint(new ArrayList<>(shapes)))); + Geometry geometry = new MultiPoint(new ArrayList<>(shapes)); + document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(geometry), new CentroidCalculator(geometry))); iw.addDocument(document); shapes.clear(); distinctHashesPerDoc.clear(); @@ -173,7 +180,9 @@ public void testGeoShapeWithSeveralDocs() throws IOException { } } if (shapes.size() != 0) { - document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, new MultiPoint(new ArrayList<>(shapes)))); + Geometry geometry = new MultiPoint(new ArrayList<>(shapes)); + document.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(geometry), new CentroidCalculator(geometry))); iw.addDocument(document); } }, geoHashGrid -> { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java index 163f8c27bdda5..f5356efd50d82 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java @@ -27,13 +27,16 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoTestUtils; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.MultiPolygon; import org.elasticsearch.geometry.Point; @@ -198,7 +201,9 @@ public void testRandomShapes() throws Exception { negRight = point.getLon(); } } - doc.add(new BinaryGeoShapeDocValuesField("field", new MultiPoint(points))); + Geometry geometry = new MultiPoint(points); + doc.add(new BinaryGeoShapeDocValuesField("field", GeoTestUtils.toDecodedTriangles(geometry), + new CentroidCalculator(geometry))); w.addDocument(doc); } GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg") @@ -337,7 +342,8 @@ public void testFiji() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { Document doc = new Document(); - doc.add(new BinaryGeoShapeDocValuesField("fiji_shape", geometryForIndexing)); + doc.add(new BinaryGeoShapeDocValuesField("fiji_shape", + GeoTestUtils.toDecodedTriangles(geometryForIndexing), new CentroidCalculator(geometryForIndexing))); for (Polygon poly : fiji) { for (int i = 0; i < poly.getPolygon().length(); i++) { doc.add(new LatLonDocValuesField("fiji_points", poly.getPolygon().getLat(i), poly.getPolygon().getLon(i))); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java index aa98274115356..4c980d185fd23 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java @@ -231,10 +231,10 @@ public void testSingleValuedFieldNearDateLineWrapLongitude() throws Exception { assertThat(bottomRight.lon(), closeTo(geoValuesBottomRight.lon(), GEOHASH_TOLERANCE)); } - // test geo_shape + // test geo_shape, should not wrap dateline { - GeoPoint geoValuesTopLeft = new GeoPoint(38, 178); - GeoPoint geoValuesBottomRight = new GeoPoint(-24, -179); + GeoPoint geoValuesTopLeft = new GeoPoint(38, -179); + GeoPoint geoValuesBottomRight = new GeoPoint(-24, 178); GeoBounds geoBounds = response.getAggregations().get(geoShapeAggName); assertThat(geoBounds, notNullValue()); assertThat(geoBounds.getName(), equalTo(geoShapeAggName)); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java index 83e3887e323f8..5b514f01ec42f 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java @@ -27,19 +27,9 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeoTestUtils; import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.GeometryCollection; -import org.elasticsearch.geometry.GeometryVisitor; -import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; -import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; @@ -168,91 +158,19 @@ public void testGeoShapeField() throws Exception { try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { GeoPoint expectedCentroid = new GeoPoint(0, 0); - CentroidCalculator centroidOfCentroidsCalculator = new CentroidCalculator(); + CompensatedSum compensatedSumLon = new CompensatedSum(0, 0); + CompensatedSum compensatedSumLat = new CompensatedSum(0, 0); for (int i = 0; i < numDocs; i++) { - CentroidCalculator calculator = new CentroidCalculator(); + Document document = new Document(); Geometry geometry = geometryGenerator.apply(false); - geometry.visit(new GeometryVisitor() { - @Override - public Void visit(Circle circle) throws Exception { - calculator.addCoordinate(circle.getX(), circle.getY()); - return null; - } - - @Override - public Void visit(GeometryCollection collection) throws Exception { - for (Geometry shape : collection) { - shape.visit(this); - } - return null; - } - - @Override - public Void visit(Line line) throws Exception { - for (int i = 0; i < line.length(); i++) { - calculator.addCoordinate(line.getX(i), line.getY(i)); - } - return null; - } - - @Override - public Void visit(LinearRing ring) throws Exception { - for (int i = 0; i < ring.length() - 1; i++) { - calculator.addCoordinate(ring.getX(i), ring.getY(i)); - } - return null; - } - - @Override - public Void visit(MultiLine multiLine) throws Exception { - for (Line line : multiLine) { - visit(line); - } - return null; - } - - @Override - public Void visit(MultiPoint multiPoint) throws Exception { - for (Point point : multiPoint) { - visit(point); - } - return null; - } - - @Override - public Void visit(MultiPolygon multiPolygon) throws Exception { - for (Polygon polygon : multiPolygon) { - visit(polygon); - } - return null; - } - - @Override - public Void visit(Point point) throws Exception { - calculator.addCoordinate(point.getX(), point.getY()); - return null; - } - - @Override - public Void visit(Polygon polygon) throws Exception { - return visit(polygon.getPolygon()); - } - - @Override - public Void visit(Rectangle rectangle) throws Exception { - calculator.addCoordinate(rectangle.getMinX(), rectangle.getMinY()); - calculator.addCoordinate(rectangle.getMinX(), rectangle.getMaxY()); - calculator.addCoordinate(rectangle.getMaxX(), rectangle.getMinY()); - calculator.addCoordinate(rectangle.getMaxX(), rectangle.getMaxY()); - return null; - } - }); - document.add(new BinaryGeoShapeDocValuesField("field", geometry)); + CentroidCalculator calculator = new CentroidCalculator(geometry); + document.add(new BinaryGeoShapeDocValuesField("field", GeoTestUtils.toDecodedTriangles(geometry), calculator)); w.addDocument(document); - centroidOfCentroidsCalculator.addCoordinate(calculator.getX(), calculator.getY()); + compensatedSumLat.add(calculator.getY()); + compensatedSumLon.add(calculator.getX()); } - expectedCentroid.reset(centroidOfCentroidsCalculator.getY(), centroidOfCentroidsCalculator.getX()); + expectedCentroid.reset(compensatedSumLat.value() / numDocs, compensatedSumLon.value() / numDocs); assertCentroid(w, expectedCentroid, new GeoShapeFieldMapper.GeoShapeFieldType()); } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java index 43f52bfd83f52..5e0e47cb7a5d5 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeIndexer.java @@ -5,10 +5,12 @@ */ package org.elasticsearch.xpack.spatial.index.mapper; +import org.apache.lucene.document.ShapeField; import org.apache.lucene.document.XYShape; import org.apache.lucene.geo.XYLine; import org.apache.lucene.geo.XYPolygon; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; @@ -54,13 +56,14 @@ public List indexShape(ParseContext context, Geometry shape) { } @Override - public void indexDocValueField(ParseContext context, Geometry shape) { + public void indexDocValueField(ParseContext context, ShapeField.DecodedTriangle[] triangles, + CentroidCalculator centroidCalculator) { BinaryGeoShapeDocValuesField docValuesField = (BinaryGeoShapeDocValuesField) context.doc().getByKey(name); if (docValuesField == null) { - docValuesField = new BinaryGeoShapeDocValuesField(name, shape); + docValuesField = new BinaryGeoShapeDocValuesField(name, triangles, centroidCalculator); context.doc().addWithKey(name, docValuesField); } else { - docValuesField.add(shape); + docValuesField.add(triangles, centroidCalculator); } } From 3337858fd0701d19ad2a4b026f96bf1cb6f9b41d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 18 Dec 2019 09:49:45 -0800 Subject: [PATCH 46/62] add shape-type metadata to geo_shape's doc-value (#50104) This commit serializes the ShapeType of the indexed geometry. The ShapeType can be useful for other future features. For one thing: #49887 depends on the ability to determine what the highest dimensional shape is for centroid calculations. GeometryCollection is reduced to the sub-shape of the highest dimension relates #37206. --- .../common/geo/CentroidCalculator.java | 13 +- .../common/geo/DimensionalShapeType.java | 177 ++++++++++++++++++ .../common/geo/TriangleTreeReader.java | 9 +- .../common/geo/TriangleTreeWriter.java | 2 +- .../index/fielddata/MultiGeoValues.java | 12 ++ .../mapper/AbstractGeometryFieldMapper.java | 1 + .../common/geo/DimensionalShapeTypeTests.java | 51 +++++ .../common/geo/TriangleTreeTests.java | 43 ++++- 8 files changed, 302 insertions(+), 6 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java create mode 100644 server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java index 51b610c02f431..92d1a2a5f752b 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java +++ b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java @@ -44,6 +44,7 @@ public class CentroidCalculator { private double sumX; private double sumY; private int count; + private DimensionalShapeType dimensionalShapeType; public CentroidCalculator(Geometry geometry) { this.sumX = 0.0; @@ -51,7 +52,9 @@ public CentroidCalculator(Geometry geometry) { this.sumY = 0.0; this.compY = 0.0; this.count = 0; - geometry.visit(new CentroidCalculatorVisitor(this)); + CentroidCalculatorVisitor visitor = new CentroidCalculatorVisitor(this); + geometry.visit(visitor); + this.dimensionalShapeType = DimensionalShapeType.forGeometry(geometry); } /** @@ -87,6 +90,7 @@ public void addFrom(CentroidCalculator otherCalculator) { addCoordinate(otherCalculator.sumX, otherCalculator.sumY); // adjust count count += otherCalculator.count - 1; + dimensionalShapeType = DimensionalShapeType.max(dimensionalShapeType, otherCalculator.dimensionalShapeType); } /** @@ -103,6 +107,10 @@ public double getY() { return sumY / count; } + public DimensionalShapeType getDimensionalShapeType() { + return dimensionalShapeType; + } + private static class CentroidCalculatorVisitor implements GeometryVisitor { private final CentroidCalculator calculator; @@ -127,7 +135,6 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - for (int i = 0; i < line.length(); i++) { calculator.addCoordinate(line.getX(i), line.getY(i)); } @@ -174,6 +181,7 @@ public Void visit(Point point) { @Override public Void visit(Polygon polygon) { + // TODO: incorporate holes into centroid calculation return visit(polygon.getPolygon()); } @@ -186,4 +194,5 @@ public Void visit(Rectangle rectangle) { return null; } } + } diff --git a/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java b/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java new file mode 100644 index 0000000000000..645010c910bba --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java @@ -0,0 +1,177 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.common.geo; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.ShapeType; + +import java.util.Comparator; + +/** + * Like {@link ShapeType} but has specific + * types for when the geometry is a {@link GeometryCollection} and + * more information about what the highest-dimensional sub-shape + * is. + */ +public enum DimensionalShapeType { + POINT, + MULTIPOINT, + LINESTRING, + MULTILINESTRING, + POLYGON, + MULTIPOLYGON, + GEOMETRYCOLLECTION_POINTS, // highest-dimensional shapes are Points + GEOMETRYCOLLECTION_LINES, // highest-dimensional shapes are Lines + GEOMETRYCOLLECTION_POLYGONS; // highest-dimensional shapes are Polygons + + private static DimensionalShapeType[] values = values(); + + private static Comparator COMPARATOR = Comparator.comparingInt(DimensionalShapeType::centroidDimension); + + public static DimensionalShapeType max(DimensionalShapeType s1, DimensionalShapeType s2) { + if (s1 == null) { + return s2; + } else if (s2 == null) { + return s1; + } + return COMPARATOR.compare(s1, s2) >= 0 ? s1 : s2; + } + + public void writeTo(ByteBuffersDataOutput out) { + out.writeByte((byte) ordinal()); + } + + public static DimensionalShapeType readFrom(ByteArrayDataInput in) { + return values[Byte.toUnsignedInt(in.readByte())]; + } + + public static DimensionalShapeType forGeometry(Geometry geometry) { + return geometry.visit(new GeometryVisitor<>() { + private DimensionalShapeType st = null; + + @Override + public DimensionalShapeType visit(Circle circle) { + st = DimensionalShapeType.max(st, DimensionalShapeType.POLYGON); + return st; + } + + @Override + public DimensionalShapeType visit(Line line) { + st = DimensionalShapeType.max(st, DimensionalShapeType.LINESTRING); + return st; + } + + @Override + public DimensionalShapeType visit(LinearRing ring) { + throw new UnsupportedOperationException("should not visit LinearRing"); + } + + @Override + public DimensionalShapeType visit(MultiLine multiLine) { + st = DimensionalShapeType.max(st, DimensionalShapeType.MULTILINESTRING); + return st; + } + + @Override + public DimensionalShapeType visit(MultiPoint multiPoint) { + st = DimensionalShapeType.max(st, DimensionalShapeType.MULTIPOINT); + return st; + } + + @Override + public DimensionalShapeType visit(MultiPolygon multiPolygon) { + st = DimensionalShapeType.max(st, DimensionalShapeType.MULTIPOLYGON); + return st; + } + + @Override + public DimensionalShapeType visit(Point point) { + st = DimensionalShapeType.max(st, DimensionalShapeType.POINT); + return st; + } + + @Override + public DimensionalShapeType visit(Polygon polygon) { + st = DimensionalShapeType.max(st, DimensionalShapeType.POLYGON); + return st; + } + + @Override + public DimensionalShapeType visit(Rectangle rectangle) { + st = DimensionalShapeType.max(st, DimensionalShapeType.POLYGON); + return st; + } + + @Override + public DimensionalShapeType visit(GeometryCollection collection) { + for (Geometry shape : collection) { + shape.visit(this); + } + int dimension = st.centroidDimension(); + if (dimension == 0) { + return DimensionalShapeType.GEOMETRYCOLLECTION_POINTS; + } else if (dimension == 1) { + return DimensionalShapeType.GEOMETRYCOLLECTION_LINES; + } else { + return DimensionalShapeType.GEOMETRYCOLLECTION_POLYGONS; + } + } + }); + } + + /** + * The integer representation of the dimension for the specific + * dimensional shape type. This is to be used by the centroid + * calculation to determine whether to add a sub-shape's centroid + * to the overall shape calculation. + * + * @return 0 for points, 1 for lines, 2 for polygons + */ + private int centroidDimension() { + switch (this) { + case POINT: + case MULTIPOINT: + case GEOMETRYCOLLECTION_POINTS: + return 0; + case LINESTRING: + case MULTILINESTRING: + case GEOMETRYCOLLECTION_LINES: + return 1; + case POLYGON: + case MULTIPOLYGON: + case GEOMETRYCOLLECTION_POLYGONS: + return 2; + default: + throw new IllegalStateException("dimension calculation of DimensionalShapeType [" + this + "] is not supported"); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index 4b346a1302cdf..f707913eb5285 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -33,8 +33,8 @@ * relations against the serialized triangle tree. */ public class TriangleTreeReader { + private static final int CENTROID_HEADER_SIZE_IN_BYTES = 9; - private static final int extentOffset = 8; private final ByteArrayDataInput input; private final CoordinateEncoder coordinateEncoder; private final Rectangle2D rectangle2D; @@ -59,7 +59,7 @@ public void reset(BytesRef bytesRef) throws IOException { public Extent getExtent() { if (treeOffset == 0) { // TODO: Compress serialization of extent - input.setPosition(extentOffset); + input.setPosition(CENTROID_HEADER_SIZE_IN_BYTES); int top = input.readInt(); int bottom = Math.toIntExact(top - input.readVLong()); int posRight = input.readInt(); @@ -90,6 +90,11 @@ public double getCentroidY() { return coordinateEncoder.decodeY(input.readInt()); } + public DimensionalShapeType getDimensionalShapeType() { + input.setPosition(8); + return DimensionalShapeType.readFrom(input); + } + /** * Compute the relation with the provided bounding box. If the result is CELL_INSIDE_QUERY * then the bounding box is within the shape. diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java index f4efa0523c7f1..135fe43c07813 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java @@ -49,7 +49,7 @@ public TriangleTreeWriter(List triangles, Coordinate public void writeTo(ByteBuffersDataOutput out) throws IOException { out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); - // TODO: Compress serialization of extent + centroidCalculator.getDimensionalShapeType().writeTo(out); out.writeInt(extent.top); out.writeVLong((long) extent.top - extent.bottom); out.writeInt(extent.posRight); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 169597f5d9572..e8140f8a98dc1 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -25,6 +25,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.CoordinateEncoder; +import org.elasticsearch.common.geo.DimensionalShapeType; import org.elasticsearch.common.geo.Extent; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoRelation; @@ -117,6 +118,11 @@ public GeoRelation relate(Rectangle rectangle) { return GeoRelation.QUERY_DISJOINT; } + @Override + public DimensionalShapeType dimensionalShapeType() { + return DimensionalShapeType.POINT; + } + @Override public double lat() { return geoPoint.lat(); @@ -162,6 +168,11 @@ public GeoRelation relate(Rectangle rectangle) { return reader.relate(minX, minY, maxX, maxY); } + @Override + public DimensionalShapeType dimensionalShapeType() { + return reader.getDimensionalShapeType(); + } + @Override public double lat() { return reader.getCentroidY(); @@ -217,6 +228,7 @@ public interface GeoValue { double lon(); BoundingBox boundingBox(); GeoRelation relate(Rectangle rectangle); + DimensionalShapeType dimensionalShapeType(); } public static class BoundingBox { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index bd3a57752115a..3c6640aa6146b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -451,6 +451,7 @@ public void parse(ParseContext context) throws IOException { System.arraycopy(bytesRef.bytes, bytesRef.offset, scratch, 0, 7 * Integer.BYTES); ShapeField.decodeTriangle(scratch, triangles[i] = new ShapeField.DecodedTriangle()); } + geometryIndexer.indexDocValueField(context, triangles, new CentroidCalculator((Geometry) shape)); } createFieldNamesField(context, fields); diff --git a/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java b/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java new file mode 100644 index 0000000000000..53297decbd83f --- /dev/null +++ b/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java @@ -0,0 +1,51 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.geo; + +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class DimensionalShapeTypeTests extends ESTestCase { + + public void testValidOrdinals() { + assertThat(DimensionalShapeType.values().length, equalTo(9)); + assertThat(DimensionalShapeType.POINT.ordinal(), equalTo(0)); + assertThat(DimensionalShapeType.MULTIPOINT.ordinal(), equalTo(1)); + assertThat(DimensionalShapeType.LINESTRING.ordinal(), equalTo(2)); + assertThat(DimensionalShapeType.MULTILINESTRING.ordinal(), equalTo(3)); + assertThat(DimensionalShapeType.POLYGON.ordinal(), equalTo(4)); + assertThat(DimensionalShapeType.MULTIPOLYGON.ordinal(), equalTo(5)); + assertThat(DimensionalShapeType.GEOMETRYCOLLECTION_POINTS.ordinal(), equalTo(6)); + assertThat(DimensionalShapeType.GEOMETRYCOLLECTION_LINES.ordinal(), equalTo(7)); + assertThat(DimensionalShapeType.GEOMETRYCOLLECTION_POLYGONS.ordinal(), equalTo(8)); + } + + public void testSerialization() { + for (DimensionalShapeType type : DimensionalShapeType.values()) { + ByteBuffersDataOutput out = new ByteBuffersDataOutput(); + type.writeTo(out); + ByteArrayDataInput input = new ByteArrayDataInput(out.toArrayCopy()); + assertThat(DimensionalShapeType.readFrom(input), equalTo(type)); + } + } +} diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java index 049ae31d06f47..494c8b1cbe1f6 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -41,11 +41,47 @@ import static org.elasticsearch.common.geo.GeoTestUtils.assertRelation; import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; import static org.elasticsearch.geo.GeometryTestUtils.fold; +import static org.elasticsearch.geo.GeometryTestUtils.randomLine; +import static org.elasticsearch.geo.GeometryTestUtils.randomMultiLine; +import static org.elasticsearch.geo.GeometryTestUtils.randomMultiPoint; +import static org.elasticsearch.geo.GeometryTestUtils.randomMultiPolygon; import static org.elasticsearch.geo.GeometryTestUtils.randomPoint; +import static org.elasticsearch.geo.GeometryTestUtils.randomPolygon; +import static org.elasticsearch.geo.GeometryTestUtils.randomRectangle; import static org.hamcrest.Matchers.equalTo; public class TriangleTreeTests extends ESTestCase { + @SuppressWarnings("unchecked") + public void testDimensionalShapeType() throws IOException { + assertDimensionalShapeType(randomPoint(false), DimensionalShapeType.POINT); + assertDimensionalShapeType(randomMultiPoint(false), DimensionalShapeType.MULTIPOINT); + assertDimensionalShapeType(randomLine(false), DimensionalShapeType.LINESTRING); + assertDimensionalShapeType(randomMultiLine(false), DimensionalShapeType.MULTILINESTRING); + assertDimensionalShapeType(randomPolygon(false), DimensionalShapeType.POLYGON); + assertDimensionalShapeType(randomMultiPolygon(false), DimensionalShapeType.MULTIPOLYGON); + assertDimensionalShapeType(randomRectangle(), DimensionalShapeType.POLYGON); + assertDimensionalShapeType(randomFrom( + new GeometryCollection<>(List.of(randomPoint(false))), + new GeometryCollection<>(List.of(randomMultiPoint(false))), + new GeometryCollection<>(Collections.singletonList( + new GeometryCollection<>(List.of(randomPoint(false), randomMultiPoint(false)))))) + , DimensionalShapeType.GEOMETRYCOLLECTION_POINTS); + assertDimensionalShapeType(randomFrom( + new GeometryCollection<>(List.of(randomPoint(false), randomLine(false))), + new GeometryCollection<>(List.of(randomMultiPoint(false), randomMultiLine(false))), + new GeometryCollection<>(Collections.singletonList( + new GeometryCollection<>(List.of(randomPoint(false), randomLine(false)))))) + , DimensionalShapeType.GEOMETRYCOLLECTION_LINES); + assertDimensionalShapeType(randomFrom( + new GeometryCollection<>(List.of(randomPoint(false), randomLine(false), randomPolygon(false))), + new GeometryCollection<>(List.of(randomMultiPoint(false), randomMultiPolygon(false))), + new GeometryCollection<>(Collections.singletonList( + new GeometryCollection<>(List.of(randomLine(false), randomPolygon(false)))))) + , DimensionalShapeType.GEOMETRYCOLLECTION_POLYGONS); + } + + public void testRectangleShape() throws IOException { for (int i = 0; i < 1000; i++) { int minX = randomIntBetween(-40, -1); @@ -214,7 +250,7 @@ public void testPacManPoints() throws Exception { public void testRandomMultiLineIntersections() throws IOException { double extentSize = randomDoubleBetween(0.01, 10, true); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - MultiLine geometry = GeometryTestUtils.randomMultiLine(false); + MultiLine geometry = randomMultiLine(false); geometry = (MultiLine) indexer.prepareForIndexing(geometry); TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); @@ -316,4 +352,9 @@ private static Geometry randomGeometryTreeCollection(int level) { } return new GeometryCollection<>(shapes); } + + private static void assertDimensionalShapeType(Geometry geometry, DimensionalShapeType expected) throws IOException { + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + assertThat(reader.getDimensionalShapeType(), equalTo(expected)); + } } From 4994a5459f1104521626937a8c414dba5aa738d4 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 19 Dec 2019 18:06:37 +0100 Subject: [PATCH 47/62] compress extent depending of how many values are set. (#50349) --- .../org/elasticsearch/common/geo/Extent.java | 99 ++++++++++++++++++- .../common/geo/TriangleTreeReader.java | 8 +- .../common/geo/TriangleTreeWriter.java | 7 +- .../elasticsearch/common/geo/ExtentTests.java | 38 +++++++ 4 files changed, 138 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index eec3d7ca408cd..227719cdf2714 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -18,6 +18,10 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteBuffersDataOutput; + +import java.io.IOException; import java.util.Objects; /** @@ -32,6 +36,13 @@ public class Extent { public int posLeft; public int posRight; + private static final byte NONE_SET = 0; + private static final byte POSITIVE_SET = 1; + private static final byte NEGATIVE_SET = 2; + private static final byte CROSSES_LAT_AXIS = 3; + private static final byte ALL_SET = 4; + + public Extent() { this.top = Integer.MIN_VALUE; this.bottom = Integer.MAX_VALUE; @@ -41,7 +52,7 @@ public Extent() { this.posRight = Integer.MIN_VALUE; } - private Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { + public Extent(int top, int bottom, int negLeft, int negRight, int posLeft, int posRight) { this.top = top; this.bottom = bottom; this.negLeft = negLeft; @@ -88,6 +99,92 @@ public void addRectangle(int bottomLeftX, int bottomLeftY, int topRightX, int to } } + static void readFromCompressed(ByteArrayDataInput input, Extent extent) { + final int top = input.readInt(); + final int bottom = Math.toIntExact(top - input.readVLong()); + final int negLeft; + final int negRight; + final int posLeft; + final int posRight; + byte type = input.readByte(); + switch (type) { + case NONE_SET: + negLeft = Integer.MAX_VALUE; + negRight = Integer.MIN_VALUE; + posLeft = Integer.MAX_VALUE; + posRight = Integer.MIN_VALUE; + break; + case POSITIVE_SET: + posRight = input.readVInt(); + posLeft = Math.toIntExact(posRight - input.readVLong()); + negLeft = Integer.MAX_VALUE; + negRight = Integer.MIN_VALUE; + break; + case NEGATIVE_SET: + negRight = input.readInt(); + negLeft = Math.toIntExact(negRight - input.readVLong()); + posLeft = Integer.MAX_VALUE; + posRight = Integer.MIN_VALUE; + break; + case CROSSES_LAT_AXIS: + posRight = input.readInt(); + negLeft = Math.toIntExact(posRight - input.readVLong()); + posLeft = 0; + negRight = 0; + break; + default: + posRight = input.readVInt(); + posLeft = Math.toIntExact(posRight - input.readVLong()); + negRight = input.readInt(); + negLeft = Math.toIntExact(negRight - input.readVLong()); + break; + } + extent.reset(top, bottom, negLeft, negRight, posLeft, posRight); + } + + void writeCompressed(ByteBuffersDataOutput output) throws IOException { + output.writeInt(this.top); + output.writeVLong((long) this.top - this.bottom); + byte type; + if (this.negLeft == Integer.MAX_VALUE && this.negRight == Integer.MIN_VALUE) { + if (this.posLeft == Integer.MAX_VALUE && this.posRight == Integer.MIN_VALUE) { + type = NONE_SET; + } else { + type = POSITIVE_SET; + } + } else if (this.posLeft == Integer.MAX_VALUE && this.posRight == Integer.MIN_VALUE) { + type = NEGATIVE_SET; + } else { + if (posLeft == 0 && negRight == 0) { + type = CROSSES_LAT_AXIS; + } else { + type = ALL_SET; + } + } + output.writeByte(type); + switch (type) { + case NONE_SET : break; + case POSITIVE_SET: + output.writeVInt(this.posRight); + output.writeVLong((long) this.posRight - this.posLeft); + break; + case NEGATIVE_SET: + output.writeInt(this.negRight); + output.writeVLong((long) this.negRight - this.negLeft); + break; + case CROSSES_LAT_AXIS: + output.writeInt(this.posRight); + output.writeVLong((long) this.posRight - this.negLeft); + break; + default: + output.writeVInt(this.posRight); + output.writeVLong((long) this.posRight - this.posLeft); + output.writeInt(this.negRight); + output.writeVLong((long) this.negRight - this.negLeft); + break; + } + } + /** * calculates the extent of a point, which is the point itself. * @param x the x-coordinate of the point diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index f707913eb5285..a1eaafa05518a 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -60,13 +60,7 @@ public Extent getExtent() { if (treeOffset == 0) { // TODO: Compress serialization of extent input.setPosition(CENTROID_HEADER_SIZE_IN_BYTES); - int top = input.readInt(); - int bottom = Math.toIntExact(top - input.readVLong()); - int posRight = input.readInt(); - int posLeft = input.readInt(); - int negRight = input.readInt(); - int negLeft = input.readInt(); - extent.reset(top, bottom, negLeft, negRight, posLeft, posRight); + Extent.readFromCompressed(input, extent); treeOffset = input.getPosition(); } else { input.setPosition(treeOffset); diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java index 135fe43c07813..15637d1d7553b 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java @@ -50,12 +50,7 @@ public void writeTo(ByteBuffersDataOutput out) throws IOException { out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); centroidCalculator.getDimensionalShapeType().writeTo(out); - out.writeInt(extent.top); - out.writeVLong((long) extent.top - extent.bottom); - out.writeInt(extent.posRight); - out.writeInt(extent.posLeft); - out.writeInt(extent.negRight); - out.writeInt(extent.negLeft); + extent.writeCompressed(out); node.writeTo(out); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java index db230df45a133..6a8feda5393ba 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java @@ -18,8 +18,15 @@ */ package org.elasticsearch.common.geo; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteBuffersDataOutput; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; + import static org.hamcrest.Matchers.equalTo; public class ExtentTests extends ESTestCase { @@ -85,4 +92,35 @@ public void testAddRectangle() { assertThat(extent.minY(), equalTo(bottomLeftY2)); assertThat(extent.maxY(), equalTo(topRightY2)); } + + public void testSerialize() throws IOException { + for (int i =0; i < 100; i++) { + Extent extent = randomExtent(); + ByteBuffersDataOutput output = new ByteBuffersDataOutput(); + extent.writeCompressed(output); + BytesRef bytesRef = new BytesRef(output.toArrayCopy(), 0, Math.toIntExact(output.size())); + ByteArrayDataInput input = new ByteArrayDataInput(); + input.reset(bytesRef.bytes, bytesRef.offset, bytesRef.length); + Extent copyExtent = new Extent(); + Extent.readFromCompressed(input, copyExtent); + assertEquals(extent, copyExtent); + } + } + + private Extent randomExtent() { + Extent extent = new Extent(); + int numberPoints = random().nextBoolean() ? 1 : randomIntBetween(2, 10); + for (int i =0; i < numberPoints; i++) { + Rectangle rectangle = GeometryTestUtils.randomRectangle(); + while (rectangle.getMinX() > rectangle.getMaxX()) { + rectangle = GeometryTestUtils.randomRectangle(); + } + int bottomLeftX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMinX()); + int bottomLeftY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMinY()); + int topRightX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMaxX()); + int topRightY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMaxY()); + extent.addRectangle(bottomLeftX, bottomLeftY, topRightX, topRightY); + } + return extent; + } } From 3794990e601f7cdaa8528bdb57a7f7237c85ffdb Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 19 Dec 2019 09:30:34 -0800 Subject: [PATCH 48/62] remove Extent.fromPoints tests --- .../elasticsearch/common/geo/ExtentTests.java | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java index 6a8feda5393ba..979f445ee8c5f 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/ExtentTests.java @@ -41,36 +41,6 @@ public void testFromPoint() { assertThat(extent.maxY(), equalTo(y)); } - public void testFromPoints() { - int bottomLeftX = randomFrom(-10, 0, 10); - int bottomLeftY = randomFrom(-10, 0, 10); - int topRightX = bottomLeftX + randomIntBetween(0, 20); - int topRightY = bottomLeftX + randomIntBetween(0, 20); - Extent extent = Extent.fromPoints(bottomLeftX, bottomLeftY, topRightX, topRightY); - assertThat(extent.minX(), equalTo(bottomLeftX)); - assertThat(extent.maxX(), equalTo(topRightX)); - assertThat(extent.minY(), equalTo(bottomLeftY)); - assertThat(extent.maxY(), equalTo(topRightY)); - assertThat(extent.top, equalTo(topRightY)); - assertThat(extent.bottom, equalTo(bottomLeftY)); - if (bottomLeftX < 0 && topRightX < 0) { - assertThat(extent.negLeft, equalTo(bottomLeftX)); - assertThat(extent.negRight, equalTo(topRightX)); - assertThat(extent.posLeft, equalTo(Integer.MAX_VALUE)); - assertThat(extent.posRight, equalTo(Integer.MIN_VALUE)); - } else if (bottomLeftX < 0) { - assertThat(extent.negLeft, equalTo(bottomLeftX)); - assertThat(extent.negRight, equalTo(bottomLeftX)); - assertThat(extent.posLeft, equalTo(topRightX)); - assertThat(extent.posRight, equalTo(topRightX)); - } else { - assertThat(extent.negLeft, equalTo(Integer.MAX_VALUE)); - assertThat(extent.negRight, equalTo(Integer.MIN_VALUE)); - assertThat(extent.posLeft, equalTo(bottomLeftX)); - assertThat(extent.posRight, equalTo(topRightX)); - } - } - public void testAddRectangle() { Extent extent = new Extent(); int bottomLeftX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(-175); From 2e15ea1e0670a072a733496decf465feb56eb76c Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 8 Jan 2020 08:34:42 -0800 Subject: [PATCH 49/62] Implement weighted geo_shape centroid support (#50297) This PR implements proper centroid calculations of geometries according to the definition defined in #49887. To compute things correctly, an additional variable encoded long representing the total weight for the centroid of the geometry in a tree. This weight is always positive. Some tests are fixed, as they did not have valid geometries. closes #49887. --- .../common/geo/CentroidCalculator.java | 107 +++++++++++++----- .../common/geo/DimensionalShapeType.java | 13 ++- .../common/geo/TriangleTreeReader.java | 28 ++++- .../common/geo/TriangleTreeWriter.java | 1 + .../index/fielddata/MultiGeoValues.java | 11 ++ .../metrics/GeoCentroidAggregator.java | 49 ++++++-- .../common/geo/CentroidCalculatorTests.java | 88 +++++++++++++- .../common/geo/GeoTestUtils.java | 12 ++ .../common/geo/TriangleTreeTests.java | 30 ++--- .../metrics/AbstractGeoTestCase.java | 5 +- .../metrics/GeoBoundsAggregatorTests.java | 19 +--- .../metrics/GeoCentroidAggregatorTests.java | 52 ++++++--- .../support/MissingValuesTests.java | 5 +- 13 files changed, 322 insertions(+), 98 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java index 92d1a2a5f752b..8df516dbab13f 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java +++ b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java @@ -38,12 +38,11 @@ * as the centroid of a shape. */ public class CentroidCalculator { - private double compX; private double compY; private double sumX; private double sumY; - private int count; + private double sumWeight; private DimensionalShapeType dimensionalShapeType; public CentroidCalculator(Geometry geometry) { @@ -51,7 +50,7 @@ public CentroidCalculator(Geometry geometry) { this.compX = 0.0; this.sumY = 0.0; this.compY = 0.0; - this.count = 0; + this.sumWeight = 0.0; CentroidCalculatorVisitor visitor = new CentroidCalculatorVisitor(this); geometry.visit(visitor); this.dimensionalShapeType = DimensionalShapeType.forGeometry(geometry); @@ -60,22 +59,22 @@ public CentroidCalculator(Geometry geometry) { /** * adds a single coordinate to the running sum and count of coordinates * for centroid calculation - * - * @param x the x-coordinate of the point + * @param x the x-coordinate of the point * @param y the y-coordinate of the point + * @param weight the associated weight of the coordinate */ - private void addCoordinate(double x, double y) { - double correctedX = x - compX; + private void addCoordinate(double x, double y, double weight) { + double correctedX = weight * x - compX; double newSumX = sumX + correctedX; compX = (newSumX - sumX) - correctedX; sumX = newSumX; - double correctedY = y - compY; + double correctedY = weight * y - compY; double newSumY = sumY + correctedY; compY = (newSumY - sumY) - correctedY; sumY = newSumY; - count += 1; + sumWeight += weight; } /** @@ -87,26 +86,45 @@ private void addCoordinate(double x, double y) { * @param otherCalculator the other centroid calculator to add from */ public void addFrom(CentroidCalculator otherCalculator) { - addCoordinate(otherCalculator.sumX, otherCalculator.sumY); - // adjust count - count += otherCalculator.count - 1; - dimensionalShapeType = DimensionalShapeType.max(dimensionalShapeType, otherCalculator.dimensionalShapeType); + int compared = DimensionalShapeType.COMPARATOR.compare(dimensionalShapeType, otherCalculator.dimensionalShapeType); + if (compared < 0) { + sumWeight = otherCalculator.sumWeight; + dimensionalShapeType = otherCalculator.dimensionalShapeType; + sumX = otherCalculator.sumX; + sumY = otherCalculator.sumY; + compX = otherCalculator.compX; + compY = otherCalculator.compY; + } else if (compared == 0) { + addCoordinate(otherCalculator.sumX, otherCalculator.sumY, otherCalculator.sumWeight); + } // else (compared > 0) do not modify centroid calculation since otherCalculator is of lower dimension than this calculator } /** * @return the x-coordinate centroid */ public double getX() { - return sumX / count; + // normalization required due to floating point precision errors + return GeoUtils.normalizeLon(sumX / sumWeight); } /** * @return the y-coordinate centroid */ public double getY() { - return sumY / count; + // normalization required due to floating point precision errors + return GeoUtils.normalizeLat(sumY / sumWeight); + } + + /** + * @return the sum of all the weighted coordinates summed in the calculator + */ + public double sumWeight() { + return sumWeight; } + /** + * @return the highest dimensional shape type summed in the calculator + */ public DimensionalShapeType getDimensionalShapeType() { return dimensionalShapeType; } @@ -121,8 +139,7 @@ private CentroidCalculatorVisitor(CentroidCalculator calculator) { @Override public Void visit(Circle circle) { - calculator.addCoordinate(circle.getX(), circle.getY()); - return null; + throw new IllegalArgumentException("invalid shape type found [Circle] while calculating centroid"); } @Override @@ -135,17 +152,47 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - for (int i = 0; i < line.length(); i++) { - calculator.addCoordinate(line.getX(i), line.getY(i)); + // a line's centroid is calculated by summing the center of each + // line segment weighted by the line segment's length in degrees + for (int i = 0; i < line.length() - 1; i++) { + double diffX = line.getX(i) - line.getX(i + 1); + double diffY = line.getY(i) - line.getY(i + 1); + double x = (line.getX(i) + line.getX(i + 1)) / 2; + double y = (line.getY(i) + line.getY(i + 1)) / 2; + calculator.addCoordinate(x, y, Math.sqrt(diffX * diffX + diffY * diffY)); } return null; } - @Override public Void visit(LinearRing ring) { + throw new IllegalArgumentException("invalid shape type found [LinearRing] while calculating centroid"); + } + + private Void visit(LinearRing ring, boolean isHole) { + // implementation of calculation defined in + // https://www.seas.upenn.edu/~sys502/extra_materials/Polygon%20Area%20and%20Centroid.pdf + // + // centroid of a ring is a weighted coordinate based on the ring's area. + // the sign of the area is positive for the outer-shell of a polygon and negative for the holes + + int sign = isHole ? -1 : 1; + double totalRingArea = 0.0; for (int i = 0; i < ring.length() - 1; i++) { - calculator.addCoordinate(ring.getX(i), ring.getY(i)); + totalRingArea += (ring.getX(i) * ring.getY(i + 1)) - (ring.getX(i + 1) * ring.getY(i)); } + totalRingArea = totalRingArea / 2; + + double sumX = 0.0; + double sumY = 0.0; + for (int i = 0; i < ring.length() - 1; i++) { + double twiceArea = (ring.getX(i) * ring.getY(i + 1)) - (ring.getX(i + 1) * ring.getY(i)); + sumX += twiceArea * (ring.getX(i) + ring.getX(i + 1)); + sumY += twiceArea * (ring.getY(i) + ring.getY(i + 1)); + } + double cX = sumX / (6 * totalRingArea); + double cY = sumY / (6 * totalRingArea); + calculator.addCoordinate(cX, cY, sign * Math.abs(totalRingArea)); + return null; } @@ -175,22 +222,26 @@ public Void visit(MultiPolygon multiPolygon) { @Override public Void visit(Point point) { - calculator.addCoordinate(point.getX(), point.getY()); + calculator.addCoordinate(point.getX(), point.getY(), 1.0); return null; } @Override public Void visit(Polygon polygon) { - // TODO: incorporate holes into centroid calculation - return visit(polygon.getPolygon()); + visit(polygon.getPolygon(), false); + for (int i = 0; i < polygon.getNumberOfHoles(); i++) { + visit(polygon.getHole(i), true); + } + return null; } @Override public Void visit(Rectangle rectangle) { - calculator.addCoordinate(rectangle.getMinX(), rectangle.getMinY()); - calculator.addCoordinate(rectangle.getMinX(), rectangle.getMaxY()); - calculator.addCoordinate(rectangle.getMaxX(), rectangle.getMinY()); - calculator.addCoordinate(rectangle.getMaxX(), rectangle.getMaxY()); + double sumX = rectangle.getMaxX() + rectangle.getMinX(); + double sumY = rectangle.getMaxY() + rectangle.getMinY(); + double diffX = rectangle.getMaxX() - rectangle.getMinX(); + double diffY = rectangle.getMaxY() - rectangle.getMinY(); + calculator.addCoordinate(sumX / 2, sumY / 2, Math.abs(diffX * diffY)); return null; } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java b/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java index 645010c910bba..db55ee4915c96 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java +++ b/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java @@ -53,9 +53,9 @@ public enum DimensionalShapeType { GEOMETRYCOLLECTION_LINES, // highest-dimensional shapes are Lines GEOMETRYCOLLECTION_POLYGONS; // highest-dimensional shapes are Polygons - private static DimensionalShapeType[] values = values(); + public static Comparator COMPARATOR = Comparator.comparingInt(DimensionalShapeType::centroidDimension); - private static Comparator COMPARATOR = Comparator.comparingInt(DimensionalShapeType::centroidDimension); + private static DimensionalShapeType[] values = values(); public static DimensionalShapeType max(DimensionalShapeType s1, DimensionalShapeType s2) { if (s1 == null) { @@ -66,12 +66,16 @@ public static DimensionalShapeType max(DimensionalShapeType s1, DimensionalShape return COMPARATOR.compare(s1, s2) >= 0 ? s1 : s2; } + public static DimensionalShapeType fromOrdinalByte(byte ordinal) { + return values[Byte.toUnsignedInt(ordinal)]; + } + public void writeTo(ByteBuffersDataOutput out) { out.writeByte((byte) ordinal()); } public static DimensionalShapeType readFrom(ByteArrayDataInput in) { - return values[Byte.toUnsignedInt(in.readByte())]; + return fromOrdinalByte(in.readByte()); } public static DimensionalShapeType forGeometry(Geometry geometry) { @@ -80,8 +84,7 @@ public static DimensionalShapeType forGeometry(Geometry geometry) { @Override public DimensionalShapeType visit(Circle circle) { - st = DimensionalShapeType.max(st, DimensionalShapeType.POLYGON); - return st; + throw new IllegalArgumentException("invalid shape type found [Circle] while computing dimensional shape type"); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index a1eaafa05518a..4dd42521743cc 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -31,10 +31,26 @@ * * This class supports checking bounding box * relations against the serialized triangle tree. + * + * ----------------------------------------- + * | The binary format of the tree | + * ----------------------------------------- + * ----------------------------------------- -- + * | centroid-x-coord (4 bytes) | | + * ----------------------------------------- | + * | centroid-y-coord (4 bytes) | | + * ----------------------------------------- | + * | DimensionalShapeType (1 byte) | | Centroid-related header + * ----------------------------------------- | + * | Sum of weights (VLong 1-8 bytes) | | + * ----------------------------------------- -- + * | Extent (var-encoding) | + * ----------------------------------------- + * | Triangle Tree | + * ----------------------------------------- + * ----------------------------------------- */ public class TriangleTreeReader { - private static final int CENTROID_HEADER_SIZE_IN_BYTES = 9; - private final ByteArrayDataInput input; private final CoordinateEncoder coordinateEncoder; private final Rectangle2D rectangle2D; @@ -58,8 +74,7 @@ public void reset(BytesRef bytesRef) throws IOException { */ public Extent getExtent() { if (treeOffset == 0) { - // TODO: Compress serialization of extent - input.setPosition(CENTROID_HEADER_SIZE_IN_BYTES); + getSumCentroidWeight(); // skip CENTROID_HEADER + var-long sum-weight Extent.readFromCompressed(input, extent); treeOffset = input.getPosition(); } else { @@ -89,6 +104,11 @@ public DimensionalShapeType getDimensionalShapeType() { return DimensionalShapeType.readFrom(input); } + public double getSumCentroidWeight() { + input.setPosition(9); + return Double.longBitsToDouble(input.readVLong()); + } + /** * Compute the relation with the provided bounding box. If the result is CELL_INSIDE_QUERY * then the bounding box is within the shape. diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java index 15637d1d7553b..e9ecb844b875b 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeWriter.java @@ -50,6 +50,7 @@ public void writeTo(ByteBuffersDataOutput out) throws IOException { out.writeInt(coordinateEncoder.encodeX(centroidCalculator.getX())); out.writeInt(coordinateEncoder.encodeY(centroidCalculator.getY())); centroidCalculator.getDimensionalShapeType().writeTo(out); + out.writeVLong(Double.doubleToLongBits(centroidCalculator.sumWeight())); extent.writeCompressed(out); node.writeTo(out); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index e8140f8a98dc1..4b15960c1ee5a 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -123,6 +123,11 @@ public DimensionalShapeType dimensionalShapeType() { return DimensionalShapeType.POINT; } + @Override + public double weight() { + return 1.0; + } + @Override public double lat() { return geoPoint.lat(); @@ -173,6 +178,11 @@ public DimensionalShapeType dimensionalShapeType() { return reader.getDimensionalShapeType(); } + @Override + public double weight() { + return reader.getSumCentroidWeight(); + } + @Override public double lat() { return reader.getCentroidY(); @@ -229,6 +239,7 @@ public interface GeoValue { BoundingBox boundingBox(); GeoRelation relate(Rectangle rectangle); DimensionalShapeType dimensionalShapeType(); + double weight(); } public static class BoundingBox { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java index 45f76ea5288a0..529195a4e861c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java @@ -20,9 +20,11 @@ package org.elasticsearch.search.aggregations.metrics; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.geo.DimensionalShapeType; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ByteArray; import org.elasticsearch.common.util.DoubleArray; import org.elasticsearch.common.util.LongArray; import org.elasticsearch.index.fielddata.MultiGeoValues; @@ -43,8 +45,9 @@ */ final class GeoCentroidAggregator extends MetricsAggregator { private final ValuesSource.Geo valuesSource; - private DoubleArray lonSum, lonCompensations, latSum, latCompensations; + private DoubleArray lonSum, lonCompensations, latSum, latCompensations, weightSum, weightCompensations; private LongArray counts; + private ByteArray dimensionalShapeTypes; GeoCentroidAggregator(String name, SearchContext context, Aggregator parent, ValuesSource.Geo valuesSource, List pipelineAggregators, @@ -57,7 +60,10 @@ final class GeoCentroidAggregator extends MetricsAggregator { lonCompensations = bigArrays.newDoubleArray(1, true); latSum = bigArrays.newDoubleArray(1, true); latCompensations = bigArrays.newDoubleArray(1, true); + weightSum = bigArrays.newDoubleArray(1, true); + weightCompensations = bigArrays.newDoubleArray(1, true); counts = bigArrays.newLongArray(1, true); + dimensionalShapeTypes = bigArrays.newByteArray(1, true); } } @@ -70,15 +76,19 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCol final MultiGeoValues values = valuesSource.geoValues(ctx); final CompensatedSum compensatedSumLat = new CompensatedSum(0, 0); final CompensatedSum compensatedSumLon = new CompensatedSum(0, 0); + final CompensatedSum compensatedSumWeight = new CompensatedSum(0, 0); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { latSum = bigArrays.grow(latSum, bucket + 1); lonSum = bigArrays.grow(lonSum, bucket + 1); + weightSum = bigArrays.grow(weightSum, bucket + 1); lonCompensations = bigArrays.grow(lonCompensations, bucket + 1); latCompensations = bigArrays.grow(latCompensations, bucket + 1); + weightCompensations = bigArrays.grow(weightCompensations, bucket + 1); counts = bigArrays.grow(counts, bucket + 1); + dimensionalShapeTypes = bigArrays.grow(dimensionalShapeTypes, bucket + 1); if (values.advanceExact(doc)) { final int valueCount = values.docValueCount(); @@ -86,26 +96,47 @@ public void collect(int doc, long bucket) throws IOException { counts.increment(bucket, valueCount); // Compute the sum of double values with Kahan summation algorithm which is more // accurate than naive summation. + DimensionalShapeType shapeType = DimensionalShapeType.fromOrdinalByte(dimensionalShapeTypes.get(bucket)); double sumLat = latSum.get(bucket); double compensationLat = latCompensations.get(bucket); double sumLon = lonSum.get(bucket); double compensationLon = lonCompensations.get(bucket); + double sumWeight = weightSum.get(bucket); + double compensatedWeight = weightCompensations.get(bucket); compensatedSumLat.reset(sumLat, compensationLat); compensatedSumLon.reset(sumLon, compensationLon); + compensatedSumWeight.reset(sumWeight, compensatedWeight); // update the sum for (int i = 0; i < valueCount; ++i) { MultiGeoValues.GeoValue value = values.nextValue(); - //latitude - compensatedSumLat.add(value.lat()); - //longitude - compensatedSumLon.add(value.lon()); + int compares = DimensionalShapeType.COMPARATOR.compare(shapeType, value.dimensionalShapeType()); + if (compares < 0) { + double coordinateWeight = value.weight(); + compensatedSumLat.reset(coordinateWeight * value.lat(), 0.0); + compensatedSumLon.reset(coordinateWeight * value.lon(), 0.0); + compensatedSumWeight.reset(coordinateWeight, 0.0); + dimensionalShapeTypes.set(bucket, (byte) value.dimensionalShapeType().ordinal()); + } else if (compares == 0) { + double coordinateWeight = value.weight(); + // weighted latitude + compensatedSumLat.add(coordinateWeight * value.lat()); + // weighted longitude + compensatedSumLon.add(coordinateWeight * value.lon()); + // weight + compensatedSumWeight.add(coordinateWeight); + } + // else (compares > 0) + // do not modify centroid calculation since shape is of lower dimension than the running dimension + } lonSum.set(bucket, compensatedSumLon.value()); lonCompensations.set(bucket, compensatedSumLon.delta()); latSum.set(bucket, compensatedSumLat.value()); latCompensations.set(bucket, compensatedSumLat.delta()); + weightSum.set(bucket, compensatedSumWeight.value()); + weightCompensations.set(bucket, compensatedSumWeight.delta()); } } }; @@ -117,8 +148,9 @@ public InternalAggregation buildAggregation(long bucket) { return buildEmptyAggregation(); } final long bucketCount = counts.get(bucket); - final GeoPoint bucketCentroid = (bucketCount > 0) - ? new GeoPoint(latSum.get(bucket) / bucketCount, lonSum.get(bucket) / bucketCount) + final double bucketWeight = weightSum.get(bucket); + final GeoPoint bucketCentroid = (bucketWeight > 0) + ? new GeoPoint(latSum.get(bucket) / bucketWeight, lonSum.get(bucket) / bucketWeight) : null; return new InternalGeoCentroid(name, bucketCentroid , bucketCount, pipelineAggregators(), metaData()); } @@ -130,6 +162,7 @@ public InternalAggregation buildEmptyAggregation() { @Override public void doClose() { - Releasables.close(latSum, latCompensations, lonSum, lonCompensations, counts); + Releasables.close(latSum, latCompensations, lonSum, lonCompensations, counts, weightSum, weightCompensations, + dimensionalShapeTypes); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java index c3933b9361310..e1d30b2b8e86a 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java @@ -18,16 +18,24 @@ */ package org.elasticsearch.common.geo; +import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; import org.elasticsearch.test.ESTestCase; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; public class CentroidCalculatorTests extends ESTestCase { + private static final double DELTA = 0.000001; - public void test() { + public void testLine() { double[] y = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; double[] x = new double[] { 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 }; double[] yRunningAvg = new double[] { 1, 1.5, 2.0, 2.5, 3, 3.5, 4, 4.5, 5, 5.5 }; @@ -44,12 +52,82 @@ public void test() { System.arraycopy(y, 0, subY, 0, i + 1); Geometry geometry = new Line(subX, subY); calculator = new CentroidCalculator(geometry); - assertThat(calculator.getX(), equalTo(xRunningAvg[i])); - assertThat(calculator.getY(), equalTo(yRunningAvg[i])); + assertEquals(xRunningAvg[i], calculator.getX(), DELTA); + assertEquals(yRunningAvg[i], calculator.getY(), DELTA); } CentroidCalculator otherCalculator = new CentroidCalculator(new Point(0, 0)); calculator.addFrom(otherCalculator); - assertThat(calculator.getX(), equalTo(50.0)); - assertThat(calculator.getY(), equalTo(5.0)); + assertEquals(55.0, calculator.getX(), DELTA); + assertEquals(5.5, calculator.getY(), DELTA); + } + + public void testRoundingErrorAndNormalization() { + double lonA = GeometryTestUtils.randomLon(); + double latA = GeometryTestUtils.randomLat(); + double lonB = randomValueOtherThanMany((l) -> Math.abs(l - lonA) <= GeoUtils.TOLERANCE, GeometryTestUtils::randomLon); + double latB = randomValueOtherThanMany((l) -> Math.abs(l - latA) <= GeoUtils.TOLERANCE, GeometryTestUtils::randomLat); + { + Line line = new Line(new double[]{180.0, 180.0}, new double[]{latA, latB}); + assertThat(new CentroidCalculator(line).getX(), anyOf(equalTo(179.99999999999997), + equalTo(180.0), equalTo(-179.99999999999997))); + } + + { + Line line = new Line(new double[]{-180.0, -180.0}, new double[]{latA, latB}); + assertThat(new CentroidCalculator(line).getX(), anyOf(equalTo(179.99999999999997), + equalTo(180.0), equalTo(-179.99999999999997))); + } + + { + Line line = new Line(new double[]{lonA, lonB}, new double[] { 90.0, 90.0 }); + assertThat(new CentroidCalculator(line).getY(), anyOf(equalTo(90.0), equalTo(89.99999999999999))); + } + + { + Line line = new Line(new double[]{lonA, lonB}, new double[] { -90.0, -90.0 }); + assertThat(new CentroidCalculator(line).getY(), anyOf(equalTo(-90.0), equalTo(-89.99999999999999))); + } + } + + // test that the centroid calculation is agnostic to orientation + public void testPolyonWithHole() { + for (boolean ccwOuter : List.of(true, false)) { + for (boolean ccwInner : List.of(true, false)) { + final LinearRing outer, inner; + if (ccwOuter) { + outer = new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}); + } else { + outer = new LinearRing(new double[]{-50, -50, 50, 50, -50}, new double[]{-50, 50, 50, -50, -50}); + } + if (ccwInner) { + inner = new LinearRing(new double[]{-40, 30, 30, -40, -40}, new double[]{-40, -40, 30, 30, -40}); + } else { + inner = new LinearRing(new double[]{-40, -40, 30, 30, -40}, new double[]{-40, 30, 30, -40, -40}); + } + final double POLY_CENTROID = 4.803921568627451; + CentroidCalculator calculator = new CentroidCalculator(new Polygon(outer, Collections.singletonList(inner))); + assertEquals(POLY_CENTROID, calculator.getX(), DELTA); + assertEquals(POLY_CENTROID, calculator.getY(), DELTA); + assertThat(calculator.sumWeight(), equalTo(5100.0)); + } + } + } + + public void testPolygonWithEqualSizedHole() { + Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), + Collections.singletonList(new LinearRing(new double[]{-50, -50, 50, 50, -50}, new double[]{-50, 50, 50, -50, -50}))); + CentroidCalculator calculator = new CentroidCalculator(polyWithHole); + assertThat(calculator.getX(), equalTo(Double.NaN)); + assertThat(calculator.getY(), equalTo(Double.NaN)); + assertThat(calculator.sumWeight(), equalTo(0.0)); + } + + public void testLineAsClosedPoint() { + double lon = GeometryTestUtils.randomLon(); + double lat = GeometryTestUtils.randomLat(); + CentroidCalculator calculator = new CentroidCalculator(new Line(new double[] {lon, lon}, new double[] { lat, lat})); + assertThat(calculator.getX(), equalTo(Double.NaN)); + assertThat(calculator.getY(), equalTo(Double.NaN)); + assertThat(calculator.sumWeight(), equalTo(0.0)); } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java index 296029464ca7d..cfbabe0357a37 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -22,11 +22,15 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.DeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.index.mapper.GeoShapeIndexer; @@ -76,4 +80,12 @@ public static String toGeoJsonString(Geometry geometry) throws IOException { GeoJson.toXContent(geometry, builder, ToXContent.EMPTY_PARAMS); return XContentHelper.convertToJson(BytesReference.bytes(builder), true, false, XContentType.JSON); } + + public static Geometry fromGeoJsonString(String geoJson) throws Exception { + XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + new BytesArray(geoJson), XContentType.JSON); + parser.nextToken(); + Geometry geometry = new GeometryParser(true, true, true).parse(parser); + return new GeoShapeIndexer(true, "indexer").prepareForIndexing(geometry); + } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java index 494c8b1cbe1f6..70adff8ee5602 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.test.ESTestCase; @@ -54,12 +55,15 @@ public class TriangleTreeTests extends ESTestCase { @SuppressWarnings("unchecked") public void testDimensionalShapeType() throws IOException { + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); assertDimensionalShapeType(randomPoint(false), DimensionalShapeType.POINT); assertDimensionalShapeType(randomMultiPoint(false), DimensionalShapeType.MULTIPOINT); assertDimensionalShapeType(randomLine(false), DimensionalShapeType.LINESTRING); assertDimensionalShapeType(randomMultiLine(false), DimensionalShapeType.MULTILINESTRING); - assertDimensionalShapeType(randomPolygon(false), DimensionalShapeType.POLYGON); - assertDimensionalShapeType(randomMultiPolygon(false), DimensionalShapeType.MULTIPOLYGON); + Geometry randoPoly = randomValueOtherThanMany(g -> g.type() != ShapeType.POLYGON, + () -> indexer.prepareForIndexing(randomPolygon(false))); + assertDimensionalShapeType(randoPoly, DimensionalShapeType.POLYGON); + assertDimensionalShapeType(indexer.prepareForIndexing(randomMultiPolygon(false)), DimensionalShapeType.MULTIPOLYGON); assertDimensionalShapeType(randomRectangle(), DimensionalShapeType.POLYGON); assertDimensionalShapeType(randomFrom( new GeometryCollection<>(List.of(randomPoint(false))), @@ -74,10 +78,12 @@ public void testDimensionalShapeType() throws IOException { new GeometryCollection<>(List.of(randomPoint(false), randomLine(false)))))) , DimensionalShapeType.GEOMETRYCOLLECTION_LINES); assertDimensionalShapeType(randomFrom( - new GeometryCollection<>(List.of(randomPoint(false), randomLine(false), randomPolygon(false))), - new GeometryCollection<>(List.of(randomMultiPoint(false), randomMultiPolygon(false))), + new GeometryCollection<>(List.of(randomPoint(false), indexer.prepareForIndexing(randomLine(false)), + indexer.prepareForIndexing(randomPolygon(false)))), + new GeometryCollection<>(List.of(randomMultiPoint(false), indexer.prepareForIndexing(randomMultiPolygon(false)))), new GeometryCollection<>(Collections.singletonList( - new GeometryCollection<>(List.of(randomLine(false), randomPolygon(false)))))) + new GeometryCollection<>(List.of(indexer.prepareForIndexing(randomLine(false)), + indexer.prepareForIndexing(randomPolygon(false))))))) , DimensionalShapeType.GEOMETRYCOLLECTION_POLYGONS); } @@ -90,8 +96,7 @@ public void testRectangleShape() throws IOException { int maxY = randomIntBetween(1, 40); double[] x = new double[]{minX, maxX, maxX, minX, minX}; double[] y = new double[]{minY, minY, maxY, maxY, minY}; - Geometry rectangle = randomBoolean() ? - new Polygon(new LinearRing(x, y), Collections.emptyList()) : new Rectangle(minX, maxX, maxY, minY); + Geometry rectangle = new Rectangle(minX, maxX, maxY, minY); TriangleTreeReader reader = triangleTreeReader(rectangle, GeoShapeCoordinateEncoder.INSTANCE); Extent expectedExtent = getExtentFromBox(minX, minY, maxX, maxY); @@ -99,8 +104,8 @@ public void testRectangleShape() throws IOException { // centroid is calculated using original double values but then loses precision as it is serialized as an integer int encodedCentroidX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(((double) minX + maxX) / 2); int encodedCentroidY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(((double) minY + maxY) / 2); - assertThat(reader.getCentroidX(), equalTo(GeoShapeCoordinateEncoder.INSTANCE.decodeX(encodedCentroidX))); - assertThat(reader.getCentroidY(), equalTo(GeoShapeCoordinateEncoder.INSTANCE.decodeY(encodedCentroidY))); + assertEquals(GeoShapeCoordinateEncoder.INSTANCE.decodeX(encodedCentroidX), reader.getCentroidX(), 0.0000001); + assertEquals(GeoShapeCoordinateEncoder.INSTANCE.decodeY(encodedCentroidY), reader.getCentroidY(), 0.0000001); // box-query touches bottom-left corner assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), @@ -153,11 +158,11 @@ public void testRectangleShape() throws IOException { public void testPacManPolygon() throws Exception { // pacman double[] px = {0, 10, 10, 0, -8, -10, -8, 0, 10, 10, 0}; - double[] py = {0, 5, 9, 10, 9, 0, -9, -10, -9, -5, 0}; + double[] py = {0, -5, -9, -10, -9, 0, 9, 10, 9, 5, 0}; // test cell crossing poly - TriangleTreeReader reader = triangleTreeReader(new Polygon(new LinearRing(py, px), Collections.emptyList()), - TestCoordinateEncoder.INSTANCE); + Polygon pacMan = new Polygon(new LinearRing(py, px), Collections.emptyList()); + TriangleTreeReader reader = triangleTreeReader(pacMan, TestCoordinateEncoder.INSTANCE); assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(2, -1, 11, 1)); assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-12, -12, 12, 12)); assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(-2, -1, 2, 0)); @@ -252,7 +257,6 @@ public void testRandomMultiLineIntersections() throws IOException { GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); MultiLine geometry = randomMultiLine(false); geometry = (MultiLine) indexer.prepareForIndexing(geometry); - TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); Extent readerExtent = reader.getExtent(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java index b302593a936d8..170afb1f6586d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.ToXContent; @@ -85,7 +86,6 @@ public void setupSuiteScopeCluster() throws Exception { multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); multiBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); singleCentroid = new GeoPoint(0, 0); - singleShapeCentroid = new GeoPoint(9.4, 34.4); multiCentroid = new GeoPoint(0, 0); unmappedCentroid = new GeoPoint(0, 0); @@ -155,7 +155,8 @@ public void setupSuiteScopeCluster() throws Exception { geoPointValues[4] = new GeoPoint(-11, 178); Line line = new Line(new double[] { 178, -179, 170, -175, 178 }, new double[] { 38, 12, -24, 32, -11 }); String lineAsWKT = new WellKnownText(false, new GeographyValidator(true)).toWKT(line); - + CentroidCalculator centroidCalculator = new CentroidCalculator(line); + singleShapeCentroid = new GeoPoint(centroidCalculator.getY(), centroidCalculator.getX()); for (int i = 0; i < 5; i++) { builders.add(client().prepareIndex(DATELINE_IDX_NAME).setSource(jsonBuilder() .startObject() diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java index f5356efd50d82..dd36df606de08 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsAggregatorTests.java @@ -26,16 +26,9 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.geo.CentroidCalculator; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoTestUtils; -import org.elasticsearch.common.geo.GeometryParser; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.MultiPolygon; @@ -44,7 +37,6 @@ import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; -import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; @@ -228,8 +220,7 @@ public void testRandomShapes() throws Exception { } public void testFiji() throws Exception { - XContentParser fijiParser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - new BytesArray("{\n" + + MultiPolygon geometryForIndexing = (MultiPolygon) GeoTestUtils.fromGeoJsonString("{\n" + " \"type\": \"MultiPolygon\",\n" + " \"coordinates\": [\n" + " [\n" + @@ -333,18 +324,14 @@ public void testFiji() throws Exception { " ]\n" + " ]\n" + " ]\n" + - " }"), XContentType.JSON); - - fijiParser.nextToken(); - MultiPolygon fiji = (MultiPolygon) new GeometryParser(true, true, true).parse(fijiParser); - MultiPolygon geometryForIndexing = (MultiPolygon) new GeoShapeIndexer(true, "indexer").prepareForIndexing(fiji); + " }"); try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { Document doc = new Document(); doc.add(new BinaryGeoShapeDocValuesField("fiji_shape", GeoTestUtils.toDecodedTriangles(geometryForIndexing), new CentroidCalculator(geometryForIndexing))); - for (Polygon poly : fiji) { + for (Polygon poly : geometryForIndexing) { for (int i = 0; i < poly.getPolygon().length(); i++) { doc.add(new LatLonDocValuesField("fiji_points", poly.getPolygon().getLat(i), poly.getPolygon().getLon(i))); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java index 5b514f01ec42f..1a645f0751474 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.store.Directory; import org.elasticsearch.common.geo.CentroidCalculator; +import org.elasticsearch.common.geo.DimensionalShapeType; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoTestUtils; import org.elasticsearch.geo.GeometryTestUtils; @@ -33,13 +34,17 @@ import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.aggregations.AggregatorTestCase; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomGeoGenerator; +import org.locationtech.spatial4j.exception.InvalidShapeException; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import java.util.function.Function; public class GeoCentroidAggregatorTests extends AggregatorTestCase { @@ -146,31 +151,46 @@ public void testMultiValuedGeoPointField() throws Exception { @SuppressWarnings("unchecked") public void testGeoShapeField() throws Exception { int numDocs = scaledRandomIntBetween(64, 256); - Function geometryGenerator = ESTestCase.randomFrom( - GeometryTestUtils::randomLine, - GeometryTestUtils::randomPoint, - GeometryTestUtils::randomPolygon, - GeometryTestUtils::randomMultiLine, - GeometryTestUtils::randomMultiPoint, - (hasAlt) -> GeometryTestUtils.randomRectangle(), - GeometryTestUtils::randomMultiPolygon - ); + List geometries = new ArrayList<>(); + DimensionalShapeType targetShapeType = null; + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + for (int i = 0; i < numDocs; i++) { + Function geometryGenerator = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPoint, + (hasAlt) -> GeometryTestUtils.randomRectangle(), + GeometryTestUtils::randomMultiPolygon + ); + Geometry geometry = geometryGenerator.apply(false); + try { + geometries.add(indexer.prepareForIndexing(geometry)); + } catch (InvalidShapeException e) { + // do not include geometry + } + targetShapeType = DimensionalShapeType.max(targetShapeType, DimensionalShapeType.forGeometry(geometry)); + } try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - GeoPoint expectedCentroid = new GeoPoint(0, 0); CompensatedSum compensatedSumLon = new CompensatedSum(0, 0); CompensatedSum compensatedSumLat = new CompensatedSum(0, 0); - for (int i = 0; i < numDocs; i++) { - + CompensatedSum compensatedSumWeight = new CompensatedSum(0, 0); + for (Geometry geometry : geometries) { Document document = new Document(); - Geometry geometry = geometryGenerator.apply(false); CentroidCalculator calculator = new CentroidCalculator(geometry); document.add(new BinaryGeoShapeDocValuesField("field", GeoTestUtils.toDecodedTriangles(geometry), calculator)); w.addDocument(document); - compensatedSumLat.add(calculator.getY()); - compensatedSumLon.add(calculator.getX()); + if (DimensionalShapeType.COMPARATOR.compare(targetShapeType, calculator.getDimensionalShapeType()) == 0) { + double weight = calculator.sumWeight(); + compensatedSumLat.add(weight * calculator.getY()); + compensatedSumLon.add(weight * calculator.getX()); + compensatedSumWeight.add(weight); + } } - expectedCentroid.reset(compensatedSumLat.value() / numDocs, compensatedSumLon.value() / numDocs); + GeoPoint expectedCentroid = new GeoPoint(compensatedSumLat.value() / compensatedSumWeight.value(), + compensatedSumLon.value() / compensatedSumWeight.value()); assertCentroid(w, expectedCentroid, new GeoShapeFieldMapper.GeoShapeFieldType()); } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java index 04edd9172eb26..0a18ae1349558 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/MissingValuesTests.java @@ -30,11 +30,13 @@ import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.TriangleTreeReader; import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.geometry.Geometry; import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; import org.elasticsearch.index.fielddata.AbstractSortedSetDocValues; import org.elasticsearch.index.fielddata.MultiGeoValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.GeoShapeIndexer; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.geo.RandomShapeGenerator; @@ -410,7 +412,8 @@ public void testMissingGeoShapes() throws IOException { values[i] = new MultiGeoValues.GeoShapeValue[random().nextInt(4)]; for (int j = 0; j < values[i].length; ++j) { ShapeBuilder builder = RandomShapeGenerator.createShape(random()); - TriangleTreeReader reader = triangleTreeReader(builder.buildGeometry(), GeoShapeCoordinateEncoder.INSTANCE); + Geometry geometry = new GeoShapeIndexer(true, "test").prepareForIndexing(builder.buildGeometry()); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); values[i][j] = new MultiGeoValues.GeoShapeValue(reader); } } From 15b029c7b38ad6ee778f1319a2434d54be55663f Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 14 Jan 2020 11:33:28 -0800 Subject: [PATCH 50/62] fix whitespace in AbstractGeometryFieldMapper loop Co-Authored-By: Adrien Grand --- .../elasticsearch/index/mapper/AbstractGeometryFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 3c6640aa6146b..fddadb513a0ed 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -445,7 +445,7 @@ public void parse(ParseContext context) throws IOException { if (fieldType().hasDocValues()) { // doc values are generated from the indexed fields. ShapeField.DecodedTriangle[] triangles = new ShapeField.DecodedTriangle[fields.size()]; - for (int i =0; i < fields.size(); i++) { + for (int i = 0; i < fields.size(); i++) { BytesRef bytesRef = fields.get(i).binaryValue(); assert bytesRef.length == 7 * Integer.BYTES; System.arraycopy(bytesRef.bytes, bytesRef.offset, scratch, 0, 7 * Integer.BYTES); From aae5d74a82f39f5cdfdc2a9e7d5d1b8b1ee5ad9d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 14 Jan 2020 11:35:09 -0800 Subject: [PATCH 51/62] use primitive double for constant in GeoTileUtils Co-Authored-By: Adrien Grand --- .../search/aggregations/bucket/geogrid/GeoTileUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 03013931f054f..74deb2144e2b5 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -43,7 +43,7 @@ */ public final class GeoTileUtils { - private static final Double PI_DIV_2 = Math.PI / 2; + private static final double PI_DIV_2 = Math.PI / 2; private GeoTileUtils() {} From 8ea0f666f311320891d4a9652b5e1584ce572308 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 14 Jan 2020 11:46:59 -0800 Subject: [PATCH 52/62] use left-shift operator in place of Math.pow in GeoGridTiler 4^(x) can be re-written as 2^(2*x) to avoid usage of the slower Math#pow method Co-Authored-By: Adrien Grand --- .../search/aggregations/bucket/geogrid/GeoGridTiler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java index eddb0ce33fddf..7ab33d26e9cd1 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -219,7 +219,7 @@ protected int setValuesByRasterization(int xTile, int yTile, int zTile, CellIdSo values.resizeCell(valuesIndex + 1); values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); } else { - values.resizeCell(valuesIndex + (int) Math.pow(4, targetPrecision - zTile) + 1); + values.resizeCell(valuesIndex + 1 << ( 2 * (targetPrecision - zTile)) + 1); valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); } } else if (GeoRelation.QUERY_CROSSES == relation) { From 7ef5fa5c42ae78e830a92fc48faa900042f49c41 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 10 Feb 2020 12:49:32 -0800 Subject: [PATCH 53/62] remove usage of Lucene's GeoRelationUtils (#52165) Lucene removed GeoRelationUtils, and so this commit inlines ES's usage of this utiity class. --- .../org/elasticsearch/index/fielddata/MultiGeoValues.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 4b15960c1ee5a..5344c0eed031b 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -20,7 +20,6 @@ import org.apache.lucene.document.ShapeField; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.spatial.util.GeoRelationUtils; import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.CentroidCalculator; @@ -111,8 +110,8 @@ public BoundingBox boundingBox() { @Override public GeoRelation relate(Rectangle rectangle) { - if (GeoRelationUtils.pointInRectPrecise(geoPoint.lat(), geoPoint.lon(), - rectangle.getMinLat(), rectangle.getMaxLat(), rectangle.getMinLon(), rectangle.getMaxLon())) { + if (geoPoint.lat() >= rectangle.getMinLat() && geoPoint.lat() <= rectangle.getMaxLat() + && geoPoint.lon() >= rectangle.getMinLon() && geoPoint.lon() <= rectangle.getMaxLon()) { return GeoRelation.QUERY_CROSSES; } return GeoRelation.QUERY_DISJOINT; From c95396db0582f664a3b1e948261430c86f8caf99 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Tue, 11 Feb 2020 10:08:52 -0800 Subject: [PATCH 54/62] GeoShape Doc Values: Fix and document tiling semantics for shapes (#52020) * Fix and document tiling semantics for shapes This commit resolves an issue in the geogrid shape tiler 1. fixes geohash brute-force-tiling to be equivalent to recursive geohash tiling 2. Resolves geotile tiling so that shapes outside of the geotile bounds are discarded 3. TriangleTree#relate is changed to be a specific relation against tiles such that intersections of tiles on the southern and western bounds of the shape are counted * more cleanup * in silico * fix a few more edge cases and mute tests for more debugging - Extent -> BoundingBox had a bug where 180/-180 and 90/-90 were treated as infinities. - awaitfixed a few edge-case tests - added muted test for checking that tile hashes of points along a tile reflect the same tiles returned by the tiler's setValues * fix checkstyle --- .../common/geo/TriangleTreeReader.java | 64 +++---- .../index/fielddata/MultiGeoValues.java | 18 +- .../bucket/geogrid/GeoGridTiler.java | 59 +++++-- .../bucket/geogrid/GeoTileUtils.java | 30 +--- .../common/geo/GeoTestUtils.java | 2 +- .../common/geo/TriangleTreeTests.java | 28 +-- .../bucket/geogrid/GeoGridTilerTests.java | 166 +++++++++++++++++- .../bucket/geogrid/GeoTileUtilsTests.java | 5 +- 8 files changed, 269 insertions(+), 103 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index 4dd42521743cc..1fcf4bdbbc57e 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -53,13 +53,13 @@ public class TriangleTreeReader { private final ByteArrayDataInput input; private final CoordinateEncoder coordinateEncoder; - private final Rectangle2D rectangle2D; + private final Tile2D tile2D; private final Extent extent; private int treeOffset; public TriangleTreeReader(CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; - this.rectangle2D = new Rectangle2D(); + this.tile2D = new Tile2D(); this.extent = new Extent(); this.input = new ByteArrayDataInput(); } @@ -113,13 +113,16 @@ public double getSumCentroidWeight() { * Compute the relation with the provided bounding box. If the result is CELL_INSIDE_QUERY * then the bounding box is within the shape. */ - public GeoRelation relate(int minX, int minY, int maxX, int maxY) { + public GeoRelation relateTile(int minX, int minY, int maxX, int maxY) { Extent extent = getExtent(); int thisMaxX = extent.maxX(); int thisMinX = extent.minX(); int thisMaxY = extent.maxY(); int thisMinY = extent.minY(); - if ((thisMinX > maxX || thisMaxX < minX || thisMinY > maxY || thisMaxY < minY)) { + + // exclude north and east boundary intersections with tiles from intersection consideration + // for consistent tiling definition of shapes on the boundaries of tiles + if ((thisMinX >= maxX || thisMaxX < minX || thisMinY > maxY || thisMaxY <= minY)) { // shapes are disjoint return GeoRelation.QUERY_DISJOINT; } @@ -129,12 +132,12 @@ public GeoRelation relate(int minX, int minY, int maxX, int maxY) { } // quick checks failed, need to traverse the tree GeoRelation rel = GeoRelation.QUERY_DISJOINT; - rectangle2D.setValues(minX, maxX, minY, maxY); + tile2D.setValues(minX, maxX, minY, maxY); byte metadata = input.readByte(); if ((metadata & 1 << 2) == 1 << 2) { // component in this node is a point int x = Math.toIntExact(thisMaxX - input.readVLong()); int y = Math.toIntExact(thisMaxY - input.readVLong()); - if (rectangle2D.contains(x, y)) { + if (tile2D.contains(x, y)) { return GeoRelation.QUERY_CROSSES; } thisMinX = x; @@ -143,7 +146,7 @@ public GeoRelation relate(int minX, int minY, int maxX, int maxY) { int aY = Math.toIntExact(thisMaxY - input.readVLong()); int bX = Math.toIntExact(thisMaxX - input.readVLong()); int bY = Math.toIntExact(thisMaxY - input.readVLong()); - if (rectangle2D.intersectsLine(aX, aY, bX, bY)) { + if (tile2D.intersectsLine(aX, aY, bX, bY)) { return GeoRelation.QUERY_CROSSES; } thisMinX = aX; @@ -157,14 +160,14 @@ public GeoRelation relate(int minX, int minY, int maxX, int maxY) { boolean ab = (metadata & 1 << 4) == 1 << 4; boolean bc = (metadata & 1 << 5) == 1 << 5; boolean ca = (metadata & 1 << 6) == 1 << 6; - rel = rectangle2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); + rel = tile2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); if (rel == GeoRelation.QUERY_CROSSES) { return GeoRelation.QUERY_CROSSES; } thisMinX = aX; } if ((metadata & 1 << 0) == 1 << 0) { // left != null - GeoRelation left = relate(rectangle2D, false, thisMaxX, thisMaxY); + GeoRelation left = relateTile(tile2D, false, thisMaxX, thisMaxY); if (left == GeoRelation.QUERY_CROSSES) { return GeoRelation.QUERY_CROSSES; } else if (left == GeoRelation.QUERY_INSIDE) { @@ -172,8 +175,8 @@ public GeoRelation relate(int minX, int minY, int maxX, int maxY) { } } if ((metadata & 1 << 1) == 1 << 1) { // right != null - if (rectangle2D.maxX >= thisMinX) { - GeoRelation right = relate(rectangle2D, false, thisMaxX, thisMaxY); + if (tile2D.maxX >= thisMinX) { + GeoRelation right = relateTile(tile2D, false, thisMaxX, thisMaxY); if (right == GeoRelation.QUERY_CROSSES) { return GeoRelation.QUERY_CROSSES; } else if (right == GeoRelation.QUERY_INSIDE) { @@ -185,19 +188,19 @@ public GeoRelation relate(int minX, int minY, int maxX, int maxY) { return rel; } - private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMaxX, int parentMaxY) { + private GeoRelation relateTile(Tile2D tile2D, boolean splitX, int parentMaxX, int parentMaxY) { int thisMaxX = Math.toIntExact(parentMaxX - input.readVLong()); int thisMaxY = Math.toIntExact(parentMaxY - input.readVLong()); GeoRelation rel = GeoRelation.QUERY_DISJOINT; int size = input.readVInt(); - if (rectangle2D.minY <= thisMaxY && rectangle2D.minX <= thisMaxX) { + if (tile2D.minY <= thisMaxY && tile2D.minX <= thisMaxX) { byte metadata = input.readByte(); int thisMinX; int thisMinY; if ((metadata & 1 << 2) == 1 << 2) { // component in this node is a point int x = Math.toIntExact(thisMaxX - input.readVLong()); int y = Math.toIntExact(thisMaxY - input.readVLong()); - if (rectangle2D.contains(x, y)) { + if (tile2D.contains(x, y)) { return GeoRelation.QUERY_CROSSES; } thisMinX = x; @@ -207,7 +210,7 @@ private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMa int aY = Math.toIntExact(thisMaxY - input.readVLong()); int bX = Math.toIntExact(thisMaxX - input.readVLong()); int bY = Math.toIntExact(thisMaxY - input.readVLong()); - if (rectangle2D.intersectsLine(aX, aY, bX, bY)) { + if (tile2D.intersectsLine(aX, aY, bX, bY)) { return GeoRelation.QUERY_CROSSES; } thisMinX = aX; @@ -222,7 +225,7 @@ private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMa boolean ab = (metadata & 1 << 4) == 1 << 4; boolean bc = (metadata & 1 << 5) == 1 << 5; boolean ca = (metadata & 1 << 6) == 1 << 6; - rel = rectangle2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); + rel = tile2D.relateTriangle(aX, aY, ab, bX, bY, bc, cX, cY, ca); if (rel == GeoRelation.QUERY_CROSSES) { return GeoRelation.QUERY_CROSSES; } @@ -230,7 +233,7 @@ private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMa thisMinY = Math.min(Math.min(aY, bY), cY); } if ((metadata & 1 << 0) == 1 << 0) { // left != null - GeoRelation left = relate(rectangle2D, !splitX, thisMaxX, thisMaxY); + GeoRelation left = relateTile(tile2D, !splitX, thisMaxX, thisMaxY); if (left == GeoRelation.QUERY_CROSSES) { return GeoRelation.QUERY_CROSSES; } else if (left == GeoRelation.QUERY_INSIDE) { @@ -239,8 +242,8 @@ private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMa } if ((metadata & 1 << 1) == 1 << 1) { // right != null int rightSize = input.readVInt(); - if ((splitX == false && rectangle2D.maxY >= thisMinY) || (splitX && rectangle2D.maxX >= thisMinX)) { - GeoRelation right = relate(rectangle2D, !splitX, thisMaxX, thisMaxY); + if ((splitX == false && tile2D.maxY >= thisMinY) || (splitX && tile2D.maxX >= thisMinX)) { + GeoRelation right = relateTile(tile2D, !splitX, thisMaxX, thisMaxY); if (right == GeoRelation.QUERY_CROSSES) { return GeoRelation.QUERY_CROSSES; } else if (right == GeoRelation.QUERY_INSIDE) { @@ -256,14 +259,14 @@ private GeoRelation relate(Rectangle2D rectangle2D, boolean splitX, int parentMa return rel; } - private static class Rectangle2D { + private static class Tile2D { protected int minX; protected int maxX; protected int minY; protected int maxY; - Rectangle2D() { + Tile2D() { } private void setValues(int minX, int maxX, int minY, int maxY) { @@ -277,7 +280,7 @@ private void setValues(int minX, int maxX, int minY, int maxY) { * Checks if the rectangle contains the provided point **/ public boolean contains(int x, int y) { - return (x < minX || x > maxX || y < minY || y > maxY) == false; + return (x <= minX || x > maxX || y < minY || y >= maxY) == false; } /** @@ -296,7 +299,7 @@ private boolean intersectsLine(int aX, int aY, int bX, int bY) { int tMaxY = StrictMath.max(aY, bY); // 2. check bounding boxes are disjoint - if (tMaxX < minX || tMinX > maxX || tMinY > maxY || tMaxY < minY) { + if (tMaxX <= minX || tMinX > maxX || tMinY > maxY || tMaxY <= minY) { return false; } @@ -311,22 +314,23 @@ private boolean intersectsLine(int aX, int aY, int bX, int bY) { * Checks if the rectangle intersects the provided triangle **/ private GeoRelation relateTriangle(int aX, int aY, boolean ab, int bX, int bY, boolean bc, int cX, int cY, boolean ca) { - // 1. query contains any triangle points - if (contains(aX, aY) || contains(bX, bY) || contains(cX, cY)) { - return GeoRelation.QUERY_CROSSES; - } - // compute bounding box of triangle int tMinX = StrictMath.min(StrictMath.min(aX, bX), cX); int tMaxX = StrictMath.max(StrictMath.max(aX, bX), cX); int tMinY = StrictMath.min(StrictMath.min(aY, bY), cY); int tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY); - // 2. check bounding boxes are disjoint - if (tMaxX < minX || tMinX > maxX || tMinY > maxY || tMaxY < minY) { + + // 1. check bounding boxes are disjoint, where north and east boundaries are not considered as crossing + if (tMaxX <= minX || tMinX > maxX || tMinY > maxY || tMaxY <= minY) { return GeoRelation.QUERY_DISJOINT; } + // 2. query contains any triangle points + if (contains(aX, aY) || contains(bX, bY) || contains(cX, cY)) { + return GeoRelation.QUERY_CROSSES; + } + boolean within = false; if (edgeIntersectsQuery(aX, aY, bX, bY)) { if (ab) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index 5344c0eed031b..ea973fa91fa5f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -169,7 +169,7 @@ public GeoRelation relate(Rectangle rectangle) { int maxX = GeoShapeCoordinateEncoder.INSTANCE.encodeX(rectangle.getMaxX()); int minY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMinY()); int maxY = GeoShapeCoordinateEncoder.INSTANCE.encodeY(rectangle.getMaxY()); - return reader.relate(minX, minY, maxX, maxY); + return reader.relateTile(minX, minY, maxX, maxY); } @Override @@ -255,24 +255,20 @@ private BoundingBox() { private void reset(Extent extent, CoordinateEncoder coordinateEncoder) { this.top = coordinateEncoder.decodeY(extent.top); this.bottom = coordinateEncoder.decodeY(extent.bottom); - if (extent.negLeft == Integer.MAX_VALUE) { + + if (extent.negLeft == Integer.MAX_VALUE && extent.negRight == Integer.MIN_VALUE) { this.negLeft = Double.POSITIVE_INFINITY; - } else { - this.negLeft = coordinateEncoder.decodeX(extent.negLeft); - } - if (extent.negRight == Integer.MIN_VALUE) { this.negRight = Double.NEGATIVE_INFINITY; } else { + this.negLeft = coordinateEncoder.decodeX(extent.negLeft); this.negRight = coordinateEncoder.decodeX(extent.negRight); } - if (extent.posLeft == Integer.MAX_VALUE) { + + if (extent.posLeft == Integer.MAX_VALUE && extent.posRight == Integer.MIN_VALUE) { this.posLeft = Double.POSITIVE_INFINITY; - } else { - this.posLeft = coordinateEncoder.decodeX(extent.posLeft); - } - if (extent.posRight == Integer.MIN_VALUE) { this.posRight = Double.NEGATIVE_INFINITY; } else { + this.posLeft = coordinateEncoder.decodeX(extent.posLeft); this.posRight = coordinateEncoder.decodeX(extent.posRight); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java index 8499a622c8c09..37a5615dcc60d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -59,6 +59,11 @@ public long encode(double x, double y, int precision) { @Override public int setValues(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, int precision) { + if (precision == 1) { + values.resizeCell(1); + values.add(0, Geohash.longEncode(0, 0, 0)); + } + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); assert bounds.minX() <= bounds.maxX(); long numLonCells = (long) ((bounds.maxX() - bounds.minX()) / Geohash.lonWidthInDegrees(precision)); @@ -72,7 +77,7 @@ public int setValues(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue } else if (count <= precision) { return setValuesByBruteForceScan(values, geoValue, precision, bounds); } else { - return setValuesByRasterization("", values, 0, precision, geoValue, bounds); + return setValuesByRasterization("", values, 0, precision, geoValue); } } @@ -83,7 +88,8 @@ protected int setValuesByBruteForceScan(GeoShapeCellValues values, MultiGeoValue int idx = 0; String min = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); String max = Geohash.stringEncode(bounds.maxX(), bounds.maxY(), precision); - double minY = Geohash.decodeLatitude(min); + String minNeighborBelow = Geohash.getNeighbor(min, precision, 0, -1); + double minY = Geohash.decodeLatitude((minNeighborBelow == null) ? min : minNeighborBelow); double minX = Geohash.decodeLongitude(min); double maxY = Geohash.decodeLatitude(max); double maxX = Geohash.decodeLongitude(max); @@ -101,15 +107,10 @@ protected int setValuesByBruteForceScan(GeoShapeCellValues values, MultiGeoValue } protected int setValuesByRasterization(String hash, GeoShapeCellValues values, int valuesIndex, - int targetPrecision, MultiGeoValues.GeoValue geoValue, - MultiGeoValues.BoundingBox shapeBounds) { + int targetPrecision, MultiGeoValues.GeoValue geoValue) { String[] hashes = Geohash.getSubGeohashes(hash); for (int i = 0; i < hashes.length; i++) { Rectangle rectangle = Geohash.toBoundingBox(hashes[i]); - if (shapeBounds.minX() == rectangle.getMaxX() || - shapeBounds.maxY() == rectangle.getMinY()) { - continue; - } GeoRelation relation = geoValue.relate(rectangle); if (relation == GeoRelation.QUERY_CROSSES) { if (hashes[i].length() == targetPrecision) { @@ -117,7 +118,7 @@ protected int setValuesByRasterization(String hash, GeoShapeCellValues values, i values.add(valuesIndex++, Geohash.longEncode(hashes[i])); } else { valuesIndex = - setValuesByRasterization(hashes[i], values, valuesIndex, targetPrecision, geoValue, shapeBounds); + setValuesByRasterization(hashes[i], values, valuesIndex, targetPrecision, geoValue); } } else if (relation == GeoRelation.QUERY_INSIDE) { if (hashes[i].length() == targetPrecision) { @@ -156,10 +157,37 @@ public long encode(double x, double y, int precision) { return GeoTileUtils.longEncode(x, y, precision); } + /** + * Sets the values of the long[] underlying {@link GeoShapeCellValues}. + * + * If the shape resides between GeoTileUtils.LATITUDE_MASK and 90 degree latitudes, then + * the shape is not accounted for since geo-tiles are only defined within those bounds. + * + * @param values the bucket values + * @param geoValue the input shape + * @param precision the tile zoom-level + * + * @return the number of tiles set by the shape + */ @Override public int setValues(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, int precision) { MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); assert bounds.minX() <= bounds.maxX(); + + if (precision == 0) { + values.resizeCell(1); + values.add(0, GeoTileUtils.longEncodeTiles(0, 0, 0)); + return 1; + } + + // geo tiles are not defined at the extreme latitudes due to them + // tiling the world as a square. + if ((bounds.top > GeoTileUtils.LATITUDE_MASK && bounds.bottom > GeoTileUtils.LATITUDE_MASK) + || (bounds.top < -GeoTileUtils.LATITUDE_MASK && bounds.bottom < -GeoTileUtils.LATITUDE_MASK)) { + return 0; + } + + final double tiles = 1 << precision; int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); @@ -173,7 +201,7 @@ public int setValues(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue } else if (count <= precision) { return setValuesByBruteForceScan(values, geoValue, precision, minXTile, minYTile, maxXTile, maxYTile); } else { - return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue, bounds); + return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue); } } @@ -200,19 +228,13 @@ protected int setValuesByBruteForceScan(GeoShapeCellValues values, MultiGeoValue } protected int setValuesByRasterization(int xTile, int yTile, int zTile, GeoShapeCellValues values, - int valuesIndex, int targetPrecision, MultiGeoValues.GeoValue geoValue, - MultiGeoValues.BoundingBox shapeBounds) { + int valuesIndex, int targetPrecision, MultiGeoValues.GeoValue geoValue) { zTile++; for (int i = 0; i < 2; i++) { for (int j = 0; j < 2; j++) { int nextX = 2 * xTile + i; int nextY = 2 * yTile + j; Rectangle rectangle = GeoTileUtils.toBoundingBox(nextX, nextY, zTile); - // TODO: this looks hacky, maybe the relate method should handle it? - if (shapeBounds.minX() == rectangle.getMaxX() || - shapeBounds.maxY() == rectangle.getMinY()) { - continue; - } GeoRelation relation = geoValue.relate(rectangle); if (GeoRelation.QUERY_INSIDE == relation) { if (zTile == targetPrecision) { @@ -227,8 +249,7 @@ protected int setValuesByRasterization(int xTile, int yTile, int zTile, GeoShape values.resizeCell(valuesIndex + 1); values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); } else { - valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, - targetPrecision, geoValue, shapeBounds); + valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, targetPrecision, geoValue); } } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 536770a34646f..4b98a4855d857 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -43,8 +43,14 @@ */ public final class GeoTileUtils { + /** + * The geo-tile map is clipped at 85.05112878 to 90 and -85.05112878 to -90 + */ + public static final double LATITUDE_MASK = 85.05112878; + private static final double PI_DIV_2 = Math.PI / 2; + private GeoTileUtils() {} /** @@ -148,30 +154,10 @@ static int getYTile(double latitude, long tiles) { */ public static long longEncode(double longitude, double latitude, int precision) { // Mathematics for this code was adapted from https://wiki.openstreetmap.org/wiki/Slippy_map_tilenames#Java - // Number of tiles for the current zoom level along the X and Y axis final long tiles = 1 << checkPrecisionRange(precision); - - long xTile = (long) Math.floor((normalizeLon(longitude) + 180) / 360 * tiles); - - double latSin = SloppyMath.cos(PI_DIV_2 - (Math.toRadians(normalizeLat(latitude)))); - long yTile = (long) Math.floor((0.5 - (Math.log((1 + latSin) / (1 - latSin)) / (4 * Math.PI))) * tiles); - - // Edge values may generate invalid values, and need to be clipped. - // For example, polar regions (above/below lat 85.05112878) get normalized. - if (xTile < 0) { - xTile = 0; - } - if (xTile >= tiles) { - xTile = tiles - 1; - } - if (yTile < 0) { - yTile = 0; - } - if (yTile >= tiles) { - yTile = tiles - 1; - } - + long xTile = getXTile(longitude, tiles); + long yTile = getYTile(latitude, tiles); return longEncodeTiles(precision, xTile, yTile); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java index cfbabe0357a37..79c81e2ee4545 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -46,7 +46,7 @@ public class GeoTestUtils { public static void assertRelation(GeoRelation expectedRelation, TriangleTreeReader reader, Extent extent) throws IOException { - GeoRelation actualRelation = reader.relate(extent.minX(), extent.minY(), extent.maxX(), extent.maxY()); + GeoRelation actualRelation = reader.relateTile(extent.minX(), extent.minY(), extent.maxX(), extent.maxY()); assertThat(actualRelation, equalTo(expectedRelation)); } diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java index 70adff8ee5602..71ac0a550c820 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -60,8 +60,14 @@ public void testDimensionalShapeType() throws IOException { assertDimensionalShapeType(randomMultiPoint(false), DimensionalShapeType.MULTIPOINT); assertDimensionalShapeType(randomLine(false), DimensionalShapeType.LINESTRING); assertDimensionalShapeType(randomMultiLine(false), DimensionalShapeType.MULTILINESTRING); - Geometry randoPoly = randomValueOtherThanMany(g -> g.type() != ShapeType.POLYGON, - () -> indexer.prepareForIndexing(randomPolygon(false))); + Geometry randoPoly = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { + try { + Geometry newGeo = indexer.prepareForIndexing(g); + return newGeo.type() != ShapeType.POLYGON; + } catch (Exception e) { + return true; + } + }, () -> randomPolygon(false))); assertDimensionalShapeType(randoPoly, DimensionalShapeType.POLYGON); assertDimensionalShapeType(indexer.prepareForIndexing(randomMultiPolygon(false)), DimensionalShapeType.MULTIPOLYGON); assertDimensionalShapeType(randomRectangle(), DimensionalShapeType.POLYGON); @@ -94,8 +100,6 @@ public void testRectangleShape() throws IOException { int maxX = randomIntBetween(1, 40); int minY = randomIntBetween(-40, -1); int maxY = randomIntBetween(1, 40); - double[] x = new double[]{minX, maxX, maxX, minX, minX}; - double[] y = new double[]{minY, minY, maxY, maxY, minY}; Geometry rectangle = new Rectangle(minX, maxX, maxY, minY); TriangleTreeReader reader = triangleTreeReader(rectangle, GeoShapeCoordinateEncoder.INSTANCE); @@ -108,17 +112,18 @@ public void testRectangleShape() throws IOException { assertEquals(GeoShapeCoordinateEncoder.INSTANCE.decodeY(encodedCentroidY), reader.getCentroidY(), 0.0000001); // box-query touches bottom-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), minY - randomIntBetween(1, 90 + minY), minX, minY)); // box-query touches bottom-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(maxX, minY - randomIntBetween(1, 90 + minY), + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(maxX, minY - randomIntBetween(1, 90 + minY), maxX + randomIntBetween(1, 180 - maxX), minY)); // box-query touches top-right corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(maxX, maxY, maxX + randomIntBetween(1, 180 - maxX), + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(maxX, maxY, maxX + randomIntBetween(1, 180 - maxX), maxY + randomIntBetween(1, 90 - maxY))); // box-query touches top-left corner - assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), maxY, minX, + assertRelation(GeoRelation.QUERY_DISJOINT, reader, getExtentFromBox(minX - randomIntBetween(1, 180 + minX), maxY, minX, maxY + randomIntBetween(1, 90 - maxY))); + // box-query fully-enclosed inside rectangle assertRelation(GeoRelation.QUERY_INSIDE, reader, getExtentFromBox(3 * (minX + maxX) / 4, 3 * (minY + maxY) / 4, 3 * (maxX + minX) / 4, 3 * (maxY + minY) / 4)); @@ -252,6 +257,7 @@ public void testPacManPoints() throws Exception { assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(xMin, yMin, xMax, yMax)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") public void testRandomMultiLineIntersections() throws IOException { double extentSize = randomDoubleBetween(0.01, 10, true); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); @@ -264,12 +270,13 @@ public void testRandomMultiLineIntersections() throws IOException { // extent that intersects edges assertRelation(GeoRelation.QUERY_CROSSES, reader, bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize)); + // TODO(talevy): resolve definition. when line is on a specific edge it is not considered crossing due to latest changes // extent that fully encloses a line in the MultiLine Extent lineExtent = triangleTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); assertRelation(GeoRelation.QUERY_CROSSES, reader, lineExtent); if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE - && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { + && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, lineExtent.maxX() + 1, lineExtent.maxY() + 1)); } @@ -285,6 +292,7 @@ public void testRandomMultiLineIntersections() throws IOException { } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") public void testRandomGeometryIntersection() throws IOException { int testPointCount = randomIntBetween(100, 200); Point[] testPoints = new Point[testPointCount]; @@ -328,7 +336,7 @@ private boolean intersects(Geometry g, Point p, double extentSize) throws IOExce Extent bufferBounds = bufferedExtentFromGeoPoint(p.getX(), p.getY(), extentSize); GeoRelation relation = triangleTreeReader(g, GeoShapeCoordinateEncoder.INSTANCE) - .relate(bufferBounds.minX(), bufferBounds.minY(), bufferBounds.maxX(), bufferBounds.maxY()); + .relateTile(bufferBounds.minX(), bufferBounds.minY(), bufferBounds.maxX(), bufferBounds.maxY()); return relation == GeoRelation.QUERY_CROSSES || relation == GeoRelation.QUERY_INSIDE; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java index 1d75fdc390d80..cf68c7e35a9c0 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -18,11 +18,19 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoBoundingBoxTests; +import org.elasticsearch.common.geo.GeoRelation; import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; +import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.TriangleTreeReader; import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.LinearRing; import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.fielddata.MultiGeoValues; @@ -30,8 +38,10 @@ import org.elasticsearch.test.ESTestCase; import java.util.Arrays; +import java.util.List; import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.LATITUDE_MASK; import static org.hamcrest.Matchers.equalTo; public class GeoGridTilerTests extends ESTestCase { @@ -72,23 +82,23 @@ public void testGeoTile() throws Exception { public void testGeoTileSetValuesBruteAndRecursiveMultiline() throws Exception { MultiLine geometry = GeometryTestUtils.randomMultiLine(false); checkGeoTileSetValuesBruteAndRecursive(geometry); - // checkGeoHashSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); } public void testGeoTileSetValuesBruteAndRecursivePolygon() throws Exception { Geometry geometry = GeometryTestUtils.randomPolygon(false); checkGeoTileSetValuesBruteAndRecursive(geometry); - // checkGeoHashSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); } public void testGeoTileSetValuesBruteAndRecursivePoints() throws Exception { Geometry geometry = randomBoolean() ? GeometryTestUtils.randomPoint(false) : GeometryTestUtils.randomMultiPoint(false); checkGeoTileSetValuesBruteAndRecursive(geometry); - // checkGeoHashSetValuesBruteAndRecursive(geometry); + checkGeoHashSetValuesBruteAndRecursive(geometry); } private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Exception { - int precision = randomIntBetween(1, 10); + int precision = randomIntBetween(1, 5); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); geometry = indexer.prepareForIndexing(geometry); TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); @@ -96,8 +106,7 @@ private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Ex GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOTILE); int recursiveCount; { - recursiveCount = GEOTILE.setValuesByRasterization(0, 0, 0, recursiveValues, 0, - precision, value, value.boundingBox()); + recursiveCount = GEOTILE.setValuesByRasterization(0, 0, 0, recursiveValues, 0, precision, value); } GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOTILE); int bruteForceCount; @@ -119,7 +128,7 @@ private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Ex } private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Exception { - int precision = randomIntBetween(1, 4); + int precision = randomIntBetween(1, 3); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); geometry = indexer.prepareForIndexing(geometry); TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); @@ -127,7 +136,7 @@ private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Ex GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOHASH); int recursiveCount; { - recursiveCount = GEOHASH.setValuesByRasterization("", recursiveValues, 0, precision, value, value.boundingBox()); + recursiveCount = GEOHASH.setValuesByRasterization("", recursiveValues, 0, precision, value); } GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOHASH); int bruteForceCount; @@ -135,7 +144,9 @@ private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Ex MultiGeoValues.BoundingBox bounds = value.boundingBox(); bruteForceCount = GEOHASH.setValuesByBruteForceScan(bruteForceValues, value, precision, bounds); } + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); Arrays.sort(recursive); @@ -143,6 +154,64 @@ private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Ex assertArrayEquals(geometry.toString(), recursive, bruteForce); } + // test random rectangles that can cross the date-line and verify that there are an expected + // number of tiles returned + public void testGeoTileSetValuesBoundingBoxes_UnboundedGeoShapeCellValues() throws Exception { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(0, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry geometry = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { + try { + indexer.prepareForIndexing(g); + return false; + } catch (Exception e) { + return true; + } + }, () -> boxToGeo(GeoBoundingBoxTests.randomBBox()))); + + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); + int expected = numTiles(value, precision); + assertThat(numTiles, equalTo(expected)); + } + } + + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") + public void testTilerMatchPoint() throws Exception { + int precision = randomIntBetween(0, 4); + Point originalPoint = GeometryTestUtils.randomPoint(false); + int xTile = GeoTileUtils.getXTile(originalPoint.getX(), 1 << precision); + int yTile = GeoTileUtils.getYTile(originalPoint.getY(), 1 << precision); + Rectangle bbox = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + long originalTileHash = GeoTileUtils.longEncode(originalPoint.getX(), originalPoint.getY(), precision); + + Point[] pointCorners = new Point[] { + // tile corners + new Point(bbox.getMinX(), bbox.getMinY()), + new Point(bbox.getMinX(), bbox.getMaxY()), + new Point(bbox.getMaxX(), bbox.getMinY()), + new Point(bbox.getMaxX(), bbox.getMaxY()), + // tile edge midpoints + new Point(bbox.getMinX(), (bbox.getMinY() + bbox.getMaxY()) / 2), + new Point(bbox.getMaxX(), (bbox.getMinY() + bbox.getMaxY()) / 2), + new Point((bbox.getMinX() + bbox.getMaxX()) / 2, bbox.getMinY()), + new Point((bbox.getMinX() + bbox.getMaxX()) / 2, bbox.getMaxY()), + }; + + for (Point point : pointCorners) { + TriangleTreeReader reader = triangleTreeReader(point, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); + assertThat(numTiles, equalTo(1)); + long tilerHash = unboundedCellValues.getValues()[0]; + long pointHash = GeoTileUtils.longEncode(point.getX(), point.getY(), precision); + assertThat(tilerHash, equalTo(pointHash)); + } + } + public void testGeoHash() throws Exception { double x = randomDouble(); double y = randomDouble(); @@ -173,4 +242,85 @@ public void testGeoHash() throws Exception { assertThat(count, equalTo(1024)); } } + + private Geometry boxToGeo(GeoBoundingBox geoBox) { + // turn into polygon + if (geoBox.right() < geoBox.left() && geoBox.right() != -180) { + return new MultiPolygon(List.of( + new Polygon(new LinearRing( + new double[] { -180, geoBox.right(), geoBox.right(), -180, -180 }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })), + new Polygon(new LinearRing( + new double[] { geoBox.left(), 180, 180, geoBox.left(), geoBox.left() }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })) + )); + } else { + double right = GeoUtils.normalizeLon(geoBox.right()); + return new Polygon(new LinearRing( + new double[] { geoBox.left(), right, right, geoBox.left(), geoBox.left() }, + new double[] { geoBox.bottom(), geoBox.bottom(), geoBox.top(), geoBox.top(), geoBox.bottom() })); + } + } + + private int numTiles(MultiGeoValues.GeoValue geoValue, int precision) { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + int count = 0; + + if (precision == 0) { + return 1; + } + + if ((bounds.top > LATITUDE_MASK && bounds.bottom > LATITUDE_MASK) + || (bounds.top < -LATITUDE_MASK && bounds.bottom < -LATITUDE_MASK)) { + return 0; + } + + final double tiles = 1 << precision; + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + if ((bounds.posLeft >= 0 && bounds.posRight >= 0) && (bounds.negLeft < 0 && bounds.negRight < 0)) { + // box one + int minXTileNeg = GeoTileUtils.getXTile(bounds.negLeft, (long) tiles); + int maxXTileNeg = GeoTileUtils.getXTile(bounds.negRight, (long) tiles); + + for (int x = minXTileNeg; x <= maxXTileNeg; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + + // box two + int minXTilePos = GeoTileUtils.getXTile(bounds.posLeft, (long) tiles); + if (minXTilePos > maxXTileNeg + 1) { + minXTilePos -= 1; + } + + int maxXTilePos = GeoTileUtils.getXTile(bounds.posRight, (long) tiles); + + for (int x = minXTilePos; x <= maxXTilePos; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } else { + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + for (int x = minXTile; x <= maxXTile; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } + } } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index 41046ca2dcd34..4090b2a5756f6 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.LATITUDE_MASK; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.hashToGeoPoint; @@ -222,8 +223,8 @@ public void testGeoTileAsLongRoutines() { * so ensure they are clipped correctly. */ public void testSingularityAtPoles() { - double minLat = -85.05112878; - double maxLat = 85.05112878; + double minLat = -LATITUDE_MASK; + double maxLat = LATITUDE_MASK; double lon = randomIntBetween(-180, 180); double lat = randomBoolean() ? randomDoubleBetween(-90, minLat, true) From 6da87aa5f6e9bbe4994fdac3e0c0d33d0dd22161 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 13 Feb 2020 09:31:04 -0800 Subject: [PATCH 55/62] fix encoding bug when clipping geo-tile latitude mask (#52274) Due to how geometries are encoded, it is important to compare the bounds of a shape to that of the encoded latitude bounds for geo-tiles. --- .../aggregations/bucket/geogrid/GeoGridTiler.java | 8 +++++--- .../aggregations/bucket/geogrid/GeoTileUtils.java | 10 +++++++++- .../org/elasticsearch/common/geo/GeoTestUtils.java | 9 +++++++++ .../bucket/geogrid/GeoGridTilerTests.java | 13 +++++++------ .../bucket/geogrid/GeoTileUtilsTests.java | 5 ++--- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java index 37a5615dcc60d..ebf7ad21d208c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -160,7 +160,8 @@ public long encode(double x, double y, int precision) { /** * Sets the values of the long[] underlying {@link GeoShapeCellValues}. * - * If the shape resides between GeoTileUtils.LATITUDE_MASK and 90 degree latitudes, then + * If the shape resides between GeoTileUtils.NORMALIZED_LATITUDE_MASK and 90 or + * between GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK and -90 degree latitudes, then * the shape is not accounted for since geo-tiles are only defined within those bounds. * * @param values the bucket values @@ -182,8 +183,9 @@ public int setValues(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue // geo tiles are not defined at the extreme latitudes due to them // tiling the world as a square. - if ((bounds.top > GeoTileUtils.LATITUDE_MASK && bounds.bottom > GeoTileUtils.LATITUDE_MASK) - || (bounds.top < -GeoTileUtils.LATITUDE_MASK && bounds.bottom < -GeoTileUtils.LATITUDE_MASK)) { + if ((bounds.top > GeoTileUtils.NORMALIZED_LATITUDE_MASK && bounds.bottom > GeoTileUtils.NORMALIZED_LATITUDE_MASK) + || (bounds.top < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK + && bounds.bottom < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK)) { return 0; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index 4b98a4855d857..c97bb88135109 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.util.SloppyMath; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.geo.GeoPoint; @@ -46,7 +47,14 @@ public final class GeoTileUtils { /** * The geo-tile map is clipped at 85.05112878 to 90 and -85.05112878 to -90 */ - public static final double LATITUDE_MASK = 85.05112878; + public static final double LATITUDE_MASK = 85.0511287798066; + + /** + * Since shapes are encoded, their boundaries are to be compared to against the encoded/decoded values of LATITUDE_MASK + */ + static final double NORMALIZED_LATITUDE_MASK = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(LATITUDE_MASK)); + static final double NORMALIZED_NEGATIVE_LATITUDE_MASK = + GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(-LATITUDE_MASK)); private static final double PI_DIV_2 = Math.PI / 2; diff --git a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java index 79c81e2ee4545..2160340dc0a64 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java +++ b/server/src/test/java/org/elasticsearch/common/geo/GeoTestUtils.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.geo; import org.apache.lucene.document.ShapeField; +import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.index.IndexableField; import org.apache.lucene.store.ByteBuffersDataOutput; import org.apache.lucene.util.BytesRef; @@ -75,6 +76,14 @@ public static TriangleTreeReader triangleTreeReader(Geometry geometry, Coordinat return reader; } + public static double encodeDecodeLat(double lat) { + return GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); + } + + public static double encodeDecodeLon(double lon) { + return GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); + } + public static String toGeoJsonString(Geometry geometry) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); GeoJson.toXContent(geometry, builder, ToXContent.EMPTY_PARAMS); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java index cf68c7e35a9c0..8d3f8b04aa7d3 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -40,8 +40,11 @@ import java.util.Arrays; import java.util.List; +import static org.elasticsearch.common.geo.GeoTestUtils.encodeDecodeLat; +import static org.elasticsearch.common.geo.GeoTestUtils.encodeDecodeLon; import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; -import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.LATITUDE_MASK; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_LATITUDE_MASK; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK; import static org.hamcrest.Matchers.equalTo; public class GeoGridTilerTests extends ESTestCase { @@ -178,14 +181,12 @@ public void testGeoTileSetValuesBoundingBoxes_UnboundedGeoShapeCellValues() thro } } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") public void testTilerMatchPoint() throws Exception { int precision = randomIntBetween(0, 4); Point originalPoint = GeometryTestUtils.randomPoint(false); int xTile = GeoTileUtils.getXTile(originalPoint.getX(), 1 << precision); int yTile = GeoTileUtils.getYTile(originalPoint.getY(), 1 << precision); Rectangle bbox = GeoTileUtils.toBoundingBox(xTile, yTile, precision); - long originalTileHash = GeoTileUtils.longEncode(originalPoint.getX(), originalPoint.getY(), precision); Point[] pointCorners = new Point[] { // tile corners @@ -207,7 +208,7 @@ public void testTilerMatchPoint() throws Exception { int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); assertThat(numTiles, equalTo(1)); long tilerHash = unboundedCellValues.getValues()[0]; - long pointHash = GeoTileUtils.longEncode(point.getX(), point.getY(), precision); + long pointHash = GeoTileUtils.longEncode(encodeDecodeLon(point.getX()), encodeDecodeLat(point.getY()), precision); assertThat(tilerHash, equalTo(pointHash)); } } @@ -270,8 +271,8 @@ private int numTiles(MultiGeoValues.GeoValue geoValue, int precision) { return 1; } - if ((bounds.top > LATITUDE_MASK && bounds.bottom > LATITUDE_MASK) - || (bounds.top < -LATITUDE_MASK && bounds.bottom < -LATITUDE_MASK)) { + if ((bounds.top > NORMALIZED_LATITUDE_MASK && bounds.bottom > NORMALIZED_LATITUDE_MASK) + || (bounds.top < NORMALIZED_NEGATIVE_LATITUDE_MASK && bounds.bottom < NORMALIZED_NEGATIVE_LATITUDE_MASK)) { return 0; } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java index 4090b2a5756f6..cf3e8699b6894 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtilsTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.LATITUDE_MASK; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.MAX_ZOOM; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.checkPrecisionRange; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.hashToGeoPoint; @@ -223,8 +222,8 @@ public void testGeoTileAsLongRoutines() { * so ensure they are clipped correctly. */ public void testSingularityAtPoles() { - double minLat = -LATITUDE_MASK; - double maxLat = LATITUDE_MASK; + double minLat = -GeoTileUtils.LATITUDE_MASK; + double maxLat = GeoTileUtils.LATITUDE_MASK; double lon = randomIntBetween(-180, 180); double lat = randomBoolean() ? randomDoubleBetween(-90, minLat, true) From 004df2ac909e37a0e0adab04c2a806112a205ad2 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 24 Feb 2020 17:28:51 +0100 Subject: [PATCH 56/62] track the offset of the BytesRef passed to the tree reader (#52704) --- .../elasticsearch/common/geo/TriangleTreeReader.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index 1fcf4bdbbc57e..cecbf227a3975 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -56,6 +56,7 @@ public class TriangleTreeReader { private final Tile2D tile2D; private final Extent extent; private int treeOffset; + private int docValueOffset; public TriangleTreeReader(CoordinateEncoder coordinateEncoder) { this.coordinateEncoder = coordinateEncoder; @@ -66,6 +67,7 @@ public TriangleTreeReader(CoordinateEncoder coordinateEncoder) { public void reset(BytesRef bytesRef) throws IOException { this.input.reset(bytesRef.bytes, bytesRef.offset, bytesRef.length); + docValueOffset = bytesRef.offset; treeOffset = 0; } @@ -87,7 +89,7 @@ public Extent getExtent() { * returns the X coordinate of the centroid. */ public double getCentroidX() { - input.setPosition(0); + input.setPosition(docValueOffset + 0); return coordinateEncoder.decodeX(input.readInt()); } @@ -95,17 +97,17 @@ public double getCentroidX() { * returns the Y coordinate of the centroid. */ public double getCentroidY() { - input.setPosition(4); + input.setPosition(docValueOffset + 4); return coordinateEncoder.decodeY(input.readInt()); } public DimensionalShapeType getDimensionalShapeType() { - input.setPosition(8); + input.setPosition(docValueOffset + 8); return DimensionalShapeType.readFrom(input); } public double getSumCentroidWeight() { - input.setPosition(9); + input.setPosition(docValueOffset + 9); return Double.longBitsToDouble(input.readVLong()); } From 54483c908ffd2e796093d97c78686affd0744163 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 24 Feb 2020 09:31:48 -0800 Subject: [PATCH 57/62] reduce bytes used when serializing Extent (#52549) This commit reflects comments made by Adrien in https://github.com/elastic/elasticsearch/pull/50834 surrounding the Extent serialization. it re-orders and negates a few values in order to save more space --- .../org/elasticsearch/common/geo/Extent.java | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/Extent.java b/server/src/main/java/org/elasticsearch/common/geo/Extent.java index 227719cdf2714..67bb96600fd82 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/Extent.java +++ b/server/src/main/java/org/elasticsearch/common/geo/Extent.java @@ -115,29 +115,31 @@ static void readFromCompressed(ByteArrayDataInput input, Extent extent) { posRight = Integer.MIN_VALUE; break; case POSITIVE_SET: - posRight = input.readVInt(); - posLeft = Math.toIntExact(posRight - input.readVLong()); + posLeft = input.readVInt(); + posRight = Math.toIntExact(input.readVLong() + posLeft); negLeft = Integer.MAX_VALUE; negRight = Integer.MIN_VALUE; break; case NEGATIVE_SET: - negRight = input.readInt(); + negRight = -input.readVInt(); negLeft = Math.toIntExact(negRight - input.readVLong()); posLeft = Integer.MAX_VALUE; posRight = Integer.MIN_VALUE; break; case CROSSES_LAT_AXIS: - posRight = input.readInt(); - negLeft = Math.toIntExact(posRight - input.readVLong()); + posRight = input.readVInt(); + negLeft = -input.readVInt(); posLeft = 0; negRight = 0; break; - default: - posRight = input.readVInt(); - posLeft = Math.toIntExact(posRight - input.readVLong()); - negRight = input.readInt(); + case ALL_SET: + posLeft = input.readVInt(); + posRight = Math.toIntExact(input.readVLong() + posLeft); + negRight = -input.readVInt(); negLeft = Math.toIntExact(negRight - input.readVLong()); break; + default: + throw new IllegalArgumentException("invalid extent values-set byte read [" + type + "]"); } extent.reset(top, bottom, negLeft, negRight, posLeft, posRight); } @@ -165,23 +167,25 @@ void writeCompressed(ByteBuffersDataOutput output) throws IOException { switch (type) { case NONE_SET : break; case POSITIVE_SET: - output.writeVInt(this.posRight); + output.writeVInt(this.posLeft); output.writeVLong((long) this.posRight - this.posLeft); break; case NEGATIVE_SET: - output.writeInt(this.negRight); + output.writeVInt(-this.negRight); output.writeVLong((long) this.negRight - this.negLeft); break; case CROSSES_LAT_AXIS: - output.writeInt(this.posRight); - output.writeVLong((long) this.posRight - this.negLeft); - break; - default: output.writeVInt(this.posRight); + output.writeVInt(-this.negLeft); + break; + case ALL_SET: + output.writeVInt(this.posLeft); output.writeVLong((long) this.posRight - this.posLeft); - output.writeInt(this.negRight); + output.writeVInt(-this.negRight); output.writeVLong((long) this.negRight - this.negLeft); break; + default: + throw new IllegalArgumentException("invalid extent values-set byte read [" + type + "]"); } } From cc289f1c6ad72825bc03c2eda9d4ff8709006023 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 24 Feb 2020 09:32:36 -0800 Subject: [PATCH 58/62] Fix geo_shape centroid calculation for downgraded shapes (#52500) This commit modifies the centroid-calculator/dimensional-shape-type to properly support the instances of polygons that have no area and lines that have no length. Beforehand N/A were returned for the centroid values, but it is best to downcast the shape type to the appropriate type. Closes #52303 --- .../common/geo/CentroidCalculator.java | 218 ++++++++++----- .../common/geo/DimensionalShapeType.java | 134 +-------- .../metrics/GeoCentroidAggregator.java | 2 +- .../common/geo/CentroidCalculatorTests.java | 264 +++++++++++++++++- .../common/geo/DimensionalShapeTypeTests.java | 12 +- .../common/geo/TriangleTreeTests.java | 14 +- .../metrics/GeoCentroidAggregatorTests.java | 11 +- 7 files changed, 415 insertions(+), 240 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java index 8df516dbab13f..466384c849b2f 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java +++ b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java @@ -31,6 +31,11 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.metrics.CompensatedSum; + +import static org.elasticsearch.common.geo.DimensionalShapeType.LINE; +import static org.elasticsearch.common.geo.DimensionalShapeType.POINT; +import static org.elasticsearch.common.geo.DimensionalShapeType.POLYGON; /** * This class keeps a running Kahan-sum of coordinates @@ -38,22 +43,20 @@ * as the centroid of a shape. */ public class CentroidCalculator { - private double compX; - private double compY; - private double sumX; - private double sumY; - private double sumWeight; + CompensatedSum compSumX; + CompensatedSum compSumY; + CompensatedSum compSumWeight; + private CentroidCalculatorVisitor visitor; private DimensionalShapeType dimensionalShapeType; public CentroidCalculator(Geometry geometry) { - this.sumX = 0.0; - this.compX = 0.0; - this.sumY = 0.0; - this.compY = 0.0; - this.sumWeight = 0.0; - CentroidCalculatorVisitor visitor = new CentroidCalculatorVisitor(this); + this.compSumX = new CompensatedSum(0, 0); + this.compSumY = new CompensatedSum(0, 0); + this.compSumWeight = new CompensatedSum(0, 0); + this.dimensionalShapeType = null; + this.visitor = new CentroidCalculatorVisitor(this); geometry.visit(visitor); - this.dimensionalShapeType = DimensionalShapeType.forGeometry(geometry); + this.dimensionalShapeType = visitor.calculator.dimensionalShapeType; } /** @@ -63,18 +66,19 @@ public CentroidCalculator(Geometry geometry) { * @param y the y-coordinate of the point * @param weight the associated weight of the coordinate */ - private void addCoordinate(double x, double y, double weight) { - double correctedX = weight * x - compX; - double newSumX = sumX + correctedX; - compX = (newSumX - sumX) - correctedX; - sumX = newSumX; - - double correctedY = weight * y - compY; - double newSumY = sumY + correctedY; - compY = (newSumY - sumY) - correctedY; - sumY = newSumY; - - sumWeight += weight; + private void addCoordinate(double x, double y, double weight, DimensionalShapeType dimensionalShapeType) { + if (this.dimensionalShapeType == null || this.dimensionalShapeType == dimensionalShapeType) { + compSumX.add(x * weight); + compSumY.add(y * weight); + compSumWeight.add(weight); + this.dimensionalShapeType = dimensionalShapeType; + } else if (dimensionalShapeType.compareTo(this.dimensionalShapeType) > 0) { + // reset counters + compSumX.reset(x * weight, 0); + compSumY.reset(y * weight, 0); + compSumWeight.reset(weight, 0); + this.dimensionalShapeType = dimensionalShapeType; + } } /** @@ -86,16 +90,17 @@ private void addCoordinate(double x, double y, double weight) { * @param otherCalculator the other centroid calculator to add from */ public void addFrom(CentroidCalculator otherCalculator) { - int compared = DimensionalShapeType.COMPARATOR.compare(dimensionalShapeType, otherCalculator.dimensionalShapeType); + int compared = dimensionalShapeType.compareTo(otherCalculator.dimensionalShapeType); if (compared < 0) { - sumWeight = otherCalculator.sumWeight; dimensionalShapeType = otherCalculator.dimensionalShapeType; - sumX = otherCalculator.sumX; - sumY = otherCalculator.sumY; - compX = otherCalculator.compX; - compY = otherCalculator.compY; + this.compSumX = otherCalculator.compSumX; + this.compSumY = otherCalculator.compSumY; + this.compSumWeight = otherCalculator.compSumWeight; + } else if (compared == 0) { - addCoordinate(otherCalculator.sumX, otherCalculator.sumY, otherCalculator.sumWeight); + this.compSumX.add(otherCalculator.compSumX.value()); + this.compSumY.add(otherCalculator.compSumY.value()); + this.compSumWeight.add(otherCalculator.compSumWeight.value()); } // else (compared > 0) do not modify centroid calculation since otherCalculator is of lower dimension than this calculator } @@ -104,7 +109,7 @@ public void addFrom(CentroidCalculator otherCalculator) { */ public double getX() { // normalization required due to floating point precision errors - return GeoUtils.normalizeLon(sumX / sumWeight); + return GeoUtils.normalizeLon(compSumX.value() / compSumWeight.value()); } /** @@ -112,14 +117,14 @@ public double getX() { */ public double getY() { // normalization required due to floating point precision errors - return GeoUtils.normalizeLat(sumY / sumWeight); + return GeoUtils.normalizeLat(compSumY.value() / compSumWeight.value()); } /** * @return the sum of all the weighted coordinates summed in the calculator */ public double sumWeight() { - return sumWeight; + return compSumWeight.value(); } /** @@ -152,62 +157,34 @@ public Void visit(GeometryCollection collection) { @Override public Void visit(Line line) { - // a line's centroid is calculated by summing the center of each - // line segment weighted by the line segment's length in degrees - for (int i = 0; i < line.length() - 1; i++) { - double diffX = line.getX(i) - line.getX(i + 1); - double diffY = line.getY(i) - line.getY(i + 1); - double x = (line.getX(i) + line.getX(i + 1)) / 2; - double y = (line.getY(i) + line.getY(i + 1)) / 2; - calculator.addCoordinate(x, y, Math.sqrt(diffX * diffX + diffY * diffY)); + if (calculator.dimensionalShapeType != POLYGON) { + visitLine(line.length(), line::getX, line::getY); } return null; } + @Override public Void visit(LinearRing ring) { throw new IllegalArgumentException("invalid shape type found [LinearRing] while calculating centroid"); } - private Void visit(LinearRing ring, boolean isHole) { - // implementation of calculation defined in - // https://www.seas.upenn.edu/~sys502/extra_materials/Polygon%20Area%20and%20Centroid.pdf - // - // centroid of a ring is a weighted coordinate based on the ring's area. - // the sign of the area is positive for the outer-shell of a polygon and negative for the holes - - int sign = isHole ? -1 : 1; - double totalRingArea = 0.0; - for (int i = 0; i < ring.length() - 1; i++) { - totalRingArea += (ring.getX(i) * ring.getY(i + 1)) - (ring.getX(i + 1) * ring.getY(i)); - } - totalRingArea = totalRingArea / 2; - - double sumX = 0.0; - double sumY = 0.0; - for (int i = 0; i < ring.length() - 1; i++) { - double twiceArea = (ring.getX(i) * ring.getY(i + 1)) - (ring.getX(i + 1) * ring.getY(i)); - sumX += twiceArea * (ring.getX(i) + ring.getX(i + 1)); - sumY += twiceArea * (ring.getY(i) + ring.getY(i + 1)); - } - double cX = sumX / (6 * totalRingArea); - double cY = sumY / (6 * totalRingArea); - calculator.addCoordinate(cX, cY, sign * Math.abs(totalRingArea)); - - return null; - } @Override public Void visit(MultiLine multiLine) { - for (Line line : multiLine) { - visit(line); + if (calculator.getDimensionalShapeType() != POLYGON) { + for (Line line : multiLine) { + visit(line); + } } return null; } @Override public Void visit(MultiPoint multiPoint) { - for (Point point : multiPoint) { - visit(point); + if (calculator.getDimensionalShapeType() == null || calculator.getDimensionalShapeType() == POINT) { + for (Point point : multiPoint) { + visit(point); + } } return null; } @@ -222,16 +199,39 @@ public Void visit(MultiPolygon multiPolygon) { @Override public Void visit(Point point) { - calculator.addCoordinate(point.getX(), point.getY(), 1.0); + if (calculator.getDimensionalShapeType() == null || calculator.getDimensionalShapeType() == POINT) { + visitPoint(point.getX(), point.getY()); + } return null; } @Override public Void visit(Polygon polygon) { - visit(polygon.getPolygon(), false); + // check area of polygon + + double[] centroidX = new double[1 + polygon.getNumberOfHoles()]; + double[] centroidY = new double[1 + polygon.getNumberOfHoles()]; + double[] weight = new double[1 + polygon.getNumberOfHoles()]; + visitLinearRing(polygon.getPolygon().length(), polygon.getPolygon()::getX, polygon.getPolygon()::getY, false, + centroidX, centroidY, weight, 0); for (int i = 0; i < polygon.getNumberOfHoles(); i++) { - visit(polygon.getHole(i), true); + visitLinearRing(polygon.getHole(i).length(), polygon.getHole(i)::getX, polygon.getHole(i)::getY, true, + centroidX, centroidY, weight, i + 1); + } + + double sumWeight = 0; + for (double w : weight) { + sumWeight += w; + } + + if (sumWeight == 0 && calculator.dimensionalShapeType != POLYGON) { + visitLine(polygon.getPolygon().length(), polygon.getPolygon()::getX, polygon.getPolygon()::getY); + } else { + for (int i = 0; i < 1 + polygon.getNumberOfHoles(); i++) { + calculator.addCoordinate(centroidX[i], centroidY[i], weight[i], POLYGON); + } } + return null; } @@ -241,9 +241,73 @@ public Void visit(Rectangle rectangle) { double sumY = rectangle.getMaxY() + rectangle.getMinY(); double diffX = rectangle.getMaxX() - rectangle.getMinX(); double diffY = rectangle.getMaxY() - rectangle.getMinY(); - calculator.addCoordinate(sumX / 2, sumY / 2, Math.abs(diffX * diffY)); + if (diffX != 0 && diffY != 0) { + calculator.addCoordinate(sumX / 2, sumY / 2, Math.abs(diffX * diffY), POLYGON); + } else if (diffX != 0) { + calculator.addCoordinate(sumX / 2, rectangle.getMinY(), diffX, LINE); + } else if (diffY != 0) { + calculator.addCoordinate(rectangle.getMinX(), sumY / 2, diffY, LINE); + } else { + visitPoint(rectangle.getMinX(), rectangle.getMinY()); + } return null; } + + + private void visitPoint(double x, double y) { + calculator.addCoordinate(x, y, 1.0, POINT); + } + + private void visitLine(int length, CoordinateSupplier x, CoordinateSupplier y) { + // check line has length + double originDiffX = x.get(0) - x.get(1); + double originDiffY = y.get(0) - y.get(1); + if (originDiffX != 0 || originDiffY != 0) { + // a line's centroid is calculated by summing the center of each + // line segment weighted by the line segment's length in degrees + for (int i = 0; i < length - 1; i++) { + double diffX = x.get(i) - x.get(i + 1); + double diffY = y.get(i) - y.get(i + 1); + double xAvg = (x.get(i) + x.get(i + 1)) / 2; + double yAvg = (y.get(i) + y.get(i + 1)) / 2; + double weight = Math.sqrt(diffX * diffX + diffY * diffY); + calculator.addCoordinate(xAvg, yAvg, weight, LINE); + } + } else { + visitPoint(x.get(0), y.get(0)); + } + } + + private void visitLinearRing(int length, CoordinateSupplier x, CoordinateSupplier y, boolean isHole, + double[] centroidX, double[] centroidY, double[] weight, int idx) { + // implementation of calculation defined in + // https://www.seas.upenn.edu/~sys502/extra_materials/Polygon%20Area%20and%20Centroid.pdf + // + // centroid of a ring is a weighted coordinate based on the ring's area. + // the sign of the area is positive for the outer-shell of a polygon and negative for the holes + + int sign = isHole ? -1 : 1; + double totalRingArea = 0.0; + for (int i = 0; i < length - 1; i++) { + totalRingArea += (x.get(i) * y.get(i + 1)) - (x.get(i + 1) * y.get(i)); + } + totalRingArea = totalRingArea / 2; + + double sumX = 0.0; + double sumY = 0.0; + for (int i = 0; i < length - 1; i++) { + double twiceArea = (x.get(i) * y.get(i + 1)) - (x.get(i + 1) * y.get(i)); + sumX += twiceArea * (x.get(i) + x.get(i + 1)); + sumY += twiceArea * (y.get(i) + y.get(i + 1)); + } + centroidX[idx] = sumX / (6 * totalRingArea); + centroidY[idx] = sumY / (6 * totalRingArea); + weight[idx] = sign * Math.abs(totalRingArea); + } } + @FunctionalInterface + private interface CoordinateSupplier { + double get(int idx); + } } diff --git a/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java b/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java index db55ee4915c96..ce9ce5a3a9d79 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java +++ b/server/src/main/java/org/elasticsearch/common/geo/DimensionalShapeType.java @@ -20,22 +20,9 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteBuffersDataOutput; -import org.elasticsearch.geometry.Circle; -import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; -import org.elasticsearch.geometry.GeometryVisitor; -import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; -import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.geometry.ShapeType; -import java.util.Comparator; - /** * Like {@link ShapeType} but has specific * types for when the geometry is a {@link GeometryCollection} and @@ -44,28 +31,11 @@ */ public enum DimensionalShapeType { POINT, - MULTIPOINT, - LINESTRING, - MULTILINESTRING, - POLYGON, - MULTIPOLYGON, - GEOMETRYCOLLECTION_POINTS, // highest-dimensional shapes are Points - GEOMETRYCOLLECTION_LINES, // highest-dimensional shapes are Lines - GEOMETRYCOLLECTION_POLYGONS; // highest-dimensional shapes are Polygons - - public static Comparator COMPARATOR = Comparator.comparingInt(DimensionalShapeType::centroidDimension); + LINE, + POLYGON; private static DimensionalShapeType[] values = values(); - public static DimensionalShapeType max(DimensionalShapeType s1, DimensionalShapeType s2) { - if (s1 == null) { - return s2; - } else if (s2 == null) { - return s1; - } - return COMPARATOR.compare(s1, s2) >= 0 ? s1 : s2; - } - public static DimensionalShapeType fromOrdinalByte(byte ordinal) { return values[Byte.toUnsignedInt(ordinal)]; } @@ -77,104 +47,4 @@ public void writeTo(ByteBuffersDataOutput out) { public static DimensionalShapeType readFrom(ByteArrayDataInput in) { return fromOrdinalByte(in.readByte()); } - - public static DimensionalShapeType forGeometry(Geometry geometry) { - return geometry.visit(new GeometryVisitor<>() { - private DimensionalShapeType st = null; - - @Override - public DimensionalShapeType visit(Circle circle) { - throw new IllegalArgumentException("invalid shape type found [Circle] while computing dimensional shape type"); - } - - @Override - public DimensionalShapeType visit(Line line) { - st = DimensionalShapeType.max(st, DimensionalShapeType.LINESTRING); - return st; - } - - @Override - public DimensionalShapeType visit(LinearRing ring) { - throw new UnsupportedOperationException("should not visit LinearRing"); - } - - @Override - public DimensionalShapeType visit(MultiLine multiLine) { - st = DimensionalShapeType.max(st, DimensionalShapeType.MULTILINESTRING); - return st; - } - - @Override - public DimensionalShapeType visit(MultiPoint multiPoint) { - st = DimensionalShapeType.max(st, DimensionalShapeType.MULTIPOINT); - return st; - } - - @Override - public DimensionalShapeType visit(MultiPolygon multiPolygon) { - st = DimensionalShapeType.max(st, DimensionalShapeType.MULTIPOLYGON); - return st; - } - - @Override - public DimensionalShapeType visit(Point point) { - st = DimensionalShapeType.max(st, DimensionalShapeType.POINT); - return st; - } - - @Override - public DimensionalShapeType visit(Polygon polygon) { - st = DimensionalShapeType.max(st, DimensionalShapeType.POLYGON); - return st; - } - - @Override - public DimensionalShapeType visit(Rectangle rectangle) { - st = DimensionalShapeType.max(st, DimensionalShapeType.POLYGON); - return st; - } - - @Override - public DimensionalShapeType visit(GeometryCollection collection) { - for (Geometry shape : collection) { - shape.visit(this); - } - int dimension = st.centroidDimension(); - if (dimension == 0) { - return DimensionalShapeType.GEOMETRYCOLLECTION_POINTS; - } else if (dimension == 1) { - return DimensionalShapeType.GEOMETRYCOLLECTION_LINES; - } else { - return DimensionalShapeType.GEOMETRYCOLLECTION_POLYGONS; - } - } - }); - } - - /** - * The integer representation of the dimension for the specific - * dimensional shape type. This is to be used by the centroid - * calculation to determine whether to add a sub-shape's centroid - * to the overall shape calculation. - * - * @return 0 for points, 1 for lines, 2 for polygons - */ - private int centroidDimension() { - switch (this) { - case POINT: - case MULTIPOINT: - case GEOMETRYCOLLECTION_POINTS: - return 0; - case LINESTRING: - case MULTILINESTRING: - case GEOMETRYCOLLECTION_LINES: - return 1; - case POLYGON: - case MULTIPOLYGON: - case GEOMETRYCOLLECTION_POLYGONS: - return 2; - default: - throw new IllegalStateException("dimension calculation of DimensionalShapeType [" + this + "] is not supported"); - } - } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java index 529195a4e861c..72e4146dd6b0d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregator.java @@ -111,7 +111,7 @@ public void collect(int doc, long bucket) throws IOException { // update the sum for (int i = 0; i < valueCount; ++i) { MultiGeoValues.GeoValue value = values.nextValue(); - int compares = DimensionalShapeType.COMPARATOR.compare(shapeType, value.dimensionalShapeType()); + int compares = shapeType.compareTo(value.dimensionalShapeType()); if (compares < 0) { double coordinateWeight = value.weight(); compensatedSumLat.reset(coordinateWeight * value.lat(), 0.0); diff --git a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java index e1d30b2b8e86a..5a5fc222debd5 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java @@ -20,20 +20,37 @@ import org.elasticsearch.geo.GeometryTestUtils; import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; import org.elasticsearch.geometry.Line; import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.test.ESTestCase; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import static org.elasticsearch.common.geo.DimensionalShapeType.LINE; +import static org.elasticsearch.common.geo.DimensionalShapeType.POINT; +import static org.elasticsearch.common.geo.DimensionalShapeType.POLYGON; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.equalTo; public class CentroidCalculatorTests extends ESTestCase { - private static final double DELTA = 0.000001; + private static final double DELTA = 0.000000001; + + public void testPoint() { + Point point = GeometryTestUtils.randomPoint(false); + CentroidCalculator calculator = new CentroidCalculator(point); + assertThat(calculator.getX(), equalTo(GeoUtils.normalizeLon(point.getX()))); + assertThat(calculator.getY(), equalTo(GeoUtils.normalizeLat(point.getY()))); + assertThat(calculator.sumWeight(), equalTo(1.0)); + assertThat(calculator.getDimensionalShapeType(), equalTo(POINT)); + } public void testLine() { double[] y = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; @@ -61,6 +78,44 @@ public void testLine() { assertEquals(5.5, calculator.getY(), DELTA); } + public void testMultiLine() { + MultiLine multiLine = GeometryTestUtils.randomMultiLine(false); + double sumLineX = 0; + double sumLineY = 0; + double sumLineWeight = 0; + for (Line line : multiLine) { + CentroidCalculator calculator = new CentroidCalculator(line); + sumLineX += calculator.compSumX.value(); + sumLineY += calculator.compSumY.value(); + sumLineWeight += calculator.compSumWeight.value(); + } + CentroidCalculator calculator = new CentroidCalculator(multiLine); + + assertEquals(sumLineX / sumLineWeight, calculator.getX(), DELTA); + assertEquals(sumLineY / sumLineWeight, calculator.getY(), DELTA); + assertEquals(sumLineWeight, calculator.sumWeight(), DELTA); + assertThat(calculator.getDimensionalShapeType(), equalTo(LINE)); + } + + public void testMultiPoint() { + MultiPoint multiPoint = GeometryTestUtils.randomMultiPoint(false); + double sumPointX = 0; + double sumPointY = 0; + double sumPointWeight = 0; + for (Point point : multiPoint) { + sumPointX += point.getX(); + sumPointY += point.getY(); + sumPointWeight += 1; + } + + CentroidCalculator calculator = new CentroidCalculator(multiPoint); + assertEquals(sumPointX / sumPointWeight, calculator.getX(), DELTA); + assertEquals(sumPointY / sumPointWeight, calculator.getY(), DELTA); + assertEquals(sumPointWeight, calculator.sumWeight(), DELTA); + assertThat(calculator.getDimensionalShapeType(), equalTo(POINT)); + + } + public void testRoundingErrorAndNormalization() { double lonA = GeometryTestUtils.randomLon(); double latA = GeometryTestUtils.randomLat(); @@ -113,21 +168,208 @@ public void testPolyonWithHole() { } } + public void testLineAsClosedPoint() { + double lon = GeometryTestUtils.randomLon(); + double lat = GeometryTestUtils.randomLat(); + CentroidCalculator calculator = new CentroidCalculator(new Line(new double[] {lon, lon}, new double[] { lat, lat})); + assertThat(calculator.getX(), equalTo(GeoUtils.normalizeLon(lon))); + assertThat(calculator.getY(), equalTo(GeoUtils.normalizeLat(lat))); + assertThat(calculator.sumWeight(), equalTo(1.0)); + } + + public void testPolygonAsLine() { + // create a line that traces itself as a polygon + Line sourceLine = GeometryTestUtils.randomLine(false); + double[] x = new double[2 * sourceLine.length() - 1]; + double[] y = new double[2 * sourceLine.length() - 1]; + int idx = 0; + for (int i = 0; i < sourceLine.length(); i++) { + x[idx] = sourceLine.getX(i); + y[idx] = sourceLine.getY(i); + idx += 1; + } + for (int i = sourceLine.length() - 2; i >= 0; i--) { + x[idx] = sourceLine.getX(i); + y[idx] = sourceLine.getY(i); + idx += 1; + } + + Line line = new Line(x, y); + CentroidCalculator lineCalculator = new CentroidCalculator(line); + + Polygon polygon = new Polygon(new LinearRing(x, y)); + CentroidCalculator calculator = new CentroidCalculator(polygon); + + // sometimes precision issues yield non-zero areas. must verify that area is close to zero + if (calculator.getDimensionalShapeType() == POLYGON) { + assertEquals(0.0, calculator.sumWeight(), 1e-10); + } else { + assertThat(calculator.getDimensionalShapeType(), equalTo(LINE)); + assertThat(calculator.getX(), equalTo(lineCalculator.getX())); + assertThat(calculator.getY(), equalTo(lineCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(lineCalculator.compSumWeight.value())); + } + } + public void testPolygonWithEqualSizedHole() { Polygon polyWithHole = new Polygon(new LinearRing(new double[]{-50, 50, 50, -50, -50}, new double[]{-50, -50, 50, 50, -50}), Collections.singletonList(new LinearRing(new double[]{-50, -50, 50, 50, -50}, new double[]{-50, 50, 50, -50, -50}))); CentroidCalculator calculator = new CentroidCalculator(polyWithHole); - assertThat(calculator.getX(), equalTo(Double.NaN)); - assertThat(calculator.getY(), equalTo(Double.NaN)); - assertThat(calculator.sumWeight(), equalTo(0.0)); + assertThat(calculator.getX(), equalTo(0.0)); + assertThat(calculator.getY(), equalTo(0.0)); + assertThat(calculator.sumWeight(), equalTo(400.0)); + assertThat(calculator.getDimensionalShapeType(), equalTo(LINE)); } - public void testLineAsClosedPoint() { - double lon = GeometryTestUtils.randomLon(); - double lat = GeometryTestUtils.randomLat(); - CentroidCalculator calculator = new CentroidCalculator(new Line(new double[] {lon, lon}, new double[] { lat, lat})); - assertThat(calculator.getX(), equalTo(Double.NaN)); - assertThat(calculator.getY(), equalTo(Double.NaN)); - assertThat(calculator.sumWeight(), equalTo(0.0)); + public void testPolygonAsPoint() { + Point point = GeometryTestUtils.randomPoint(false); + Polygon polygon = new Polygon(new LinearRing(new double[] { point.getX(), point.getX(), point.getX(), point.getX() }, + new double[] { point.getY(), point.getY(), point.getY(), point.getY() })); + CentroidCalculator calculator = new CentroidCalculator(polygon); + assertThat(calculator.getX(), equalTo(GeoUtils.normalizeLon(point.getX()))); + assertThat(calculator.getY(), equalTo(GeoUtils.normalizeLat(point.getY()))); + assertThat(calculator.sumWeight(), equalTo(1.0)); + assertThat(calculator.getDimensionalShapeType(), equalTo(POINT)); + } + + public void testGeometryCollection() { + int numPoints = randomIntBetween(0, 3); + int numLines = randomIntBetween(0, 3); + int numPolygons = randomIntBetween(0, 3); + + if (numPoints == 0 && numLines == 0 && numPolygons == 0) { + numPoints = 1; + numLines = 1; + numPolygons = 1; + } + List shapes = new ArrayList<>(); + for (int i = 0; i < numPoints; i++) { + if (randomBoolean()) { + shapes.add(GeometryTestUtils.randomPoint(false)); + } else { + shapes.add(GeometryTestUtils.randomMultiPoint(false)); + } + } + for (int i = 0; i < numLines; i++) { + if (randomBoolean()) { + shapes.add(GeometryTestUtils.randomLine(false)); + } else { + shapes.add(GeometryTestUtils.randomMultiLine(false)); + } + } + for (int i = 0; i < numPolygons; i++) { + if (randomBoolean()) { + shapes.add(GeometryTestUtils.randomPolygon(false)); + } else { + shapes.add(GeometryTestUtils.randomMultiPolygon(false)); + } + } + + DimensionalShapeType dimensionalShapeType = numPolygons > 0 ? POLYGON : numLines > 0 ? LINE : POINT; + + // addFromCalculator is only adding from shapes with the highest dimensionalShapeType + CentroidCalculator addFromCalculator = null; + for (Geometry shape : shapes) { + if ((shape.type() == ShapeType.MULTIPOLYGON || shape.type() == ShapeType.POLYGON) || + (dimensionalShapeType == LINE && (shape.type() == ShapeType.LINESTRING || shape.type() == ShapeType.MULTILINESTRING)) || + (dimensionalShapeType == POINT && (shape.type() == ShapeType.POINT || shape.type() == ShapeType.MULTIPOINT))) { + if (addFromCalculator == null) { + addFromCalculator = new CentroidCalculator(shape); + } else { + addFromCalculator.addFrom(new CentroidCalculator(shape)); + } + } + } + + // shuffle + if (randomBoolean()) { + Collections.shuffle(shapes, random()); + } else if (randomBoolean()) { + Collections.reverse(shapes); + } + + GeometryCollection collection = new GeometryCollection<>(shapes); + CentroidCalculator calculator = new CentroidCalculator(collection); + + assertThat(addFromCalculator.getDimensionalShapeType(), equalTo(dimensionalShapeType)); + assertThat(calculator.getDimensionalShapeType(), equalTo(dimensionalShapeType)); + assertEquals(calculator.getX(), addFromCalculator.getX(), DELTA); + assertEquals(calculator.getY(), addFromCalculator.getY(), DELTA); + assertEquals(calculator.sumWeight(), addFromCalculator.sumWeight(), DELTA); + } + + public void testAddFrom() { + Point point = GeometryTestUtils.randomPoint(false); + Line line = GeometryTestUtils.randomLine(false); + Polygon polygon = GeometryTestUtils.randomPolygon(false); + + // point add point + { + CentroidCalculator calculator = new CentroidCalculator(point); + calculator.addFrom(new CentroidCalculator(point)); + assertThat(calculator.compSumX.value(), equalTo(2 * point.getX())); + assertThat(calculator.compSumY.value(), equalTo(2 * point.getY())); + assertThat(calculator.sumWeight(), equalTo(2.0)); + } + + // point add line/polygon + { + CentroidCalculator lineCalculator = new CentroidCalculator(line); + CentroidCalculator calculator = new CentroidCalculator(point); + calculator.addFrom(lineCalculator); + assertThat(calculator.getX(), equalTo(lineCalculator.getX())); + assertThat(calculator.getY(), equalTo(lineCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(lineCalculator.sumWeight())); + } + + // line add point + { + CentroidCalculator lineCalculator = new CentroidCalculator(line); + CentroidCalculator calculator = new CentroidCalculator(line); + calculator.addFrom(new CentroidCalculator(point)); + assertThat(calculator.getX(), equalTo(lineCalculator.getX())); + assertThat(calculator.getY(), equalTo(lineCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(lineCalculator.sumWeight())); + } + + // line add line + { + CentroidCalculator lineCalculator = new CentroidCalculator(line); + CentroidCalculator calculator = new CentroidCalculator(line); + calculator.addFrom(lineCalculator); + assertEquals(2 * lineCalculator.compSumX.value(), calculator.compSumX.value(), DELTA); + assertEquals(2 * lineCalculator.compSumY.value(), calculator.compSumY.value(), DELTA); + assertEquals(2 * lineCalculator.sumWeight(), calculator.sumWeight(), DELTA); + } + + // line add polygon + { + CentroidCalculator polygonCalculator = new CentroidCalculator(polygon); + CentroidCalculator calculator = new CentroidCalculator(line); + calculator.addFrom(polygonCalculator); + assertThat(calculator.getX(), equalTo(polygonCalculator.getX())); + assertThat(calculator.getY(), equalTo(polygonCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(calculator.sumWeight())); + } + + // polygon add point/line + { + CentroidCalculator polygonCalculator = new CentroidCalculator(polygon); + CentroidCalculator calculator = new CentroidCalculator(polygon); + calculator.addFrom(new CentroidCalculator(randomBoolean() ? point : line)); + assertThat(calculator.getX(), equalTo(polygonCalculator.getX())); + assertThat(calculator.getY(), equalTo(polygonCalculator.getY())); + assertThat(calculator.sumWeight(), equalTo(calculator.sumWeight())); + } + + // polygon add polygon + { + CentroidCalculator polygonCalculator = new CentroidCalculator(polygon); + CentroidCalculator calculator = new CentroidCalculator(polygon); + calculator.addFrom(polygonCalculator); + assertThat(calculator.compSumX.value(), equalTo(2 * polygonCalculator.compSumX.value())); + assertThat(calculator.compSumY.value(), equalTo(2 * polygonCalculator.compSumY.value())); + assertThat(calculator.sumWeight(), equalTo(2 * polygonCalculator.sumWeight())); + } } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java b/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java index 53297decbd83f..9966f87dc97e2 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/DimensionalShapeTypeTests.java @@ -28,16 +28,10 @@ public class DimensionalShapeTypeTests extends ESTestCase { public void testValidOrdinals() { - assertThat(DimensionalShapeType.values().length, equalTo(9)); + assertThat(DimensionalShapeType.values().length, equalTo(3)); assertThat(DimensionalShapeType.POINT.ordinal(), equalTo(0)); - assertThat(DimensionalShapeType.MULTIPOINT.ordinal(), equalTo(1)); - assertThat(DimensionalShapeType.LINESTRING.ordinal(), equalTo(2)); - assertThat(DimensionalShapeType.MULTILINESTRING.ordinal(), equalTo(3)); - assertThat(DimensionalShapeType.POLYGON.ordinal(), equalTo(4)); - assertThat(DimensionalShapeType.MULTIPOLYGON.ordinal(), equalTo(5)); - assertThat(DimensionalShapeType.GEOMETRYCOLLECTION_POINTS.ordinal(), equalTo(6)); - assertThat(DimensionalShapeType.GEOMETRYCOLLECTION_LINES.ordinal(), equalTo(7)); - assertThat(DimensionalShapeType.GEOMETRYCOLLECTION_POLYGONS.ordinal(), equalTo(8)); + assertThat(DimensionalShapeType.LINE.ordinal(), equalTo(1)); + assertThat(DimensionalShapeType.POLYGON.ordinal(), equalTo(2)); } public void testSerialization() { diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java index 71ac0a550c820..3de9d70d459ab 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -57,9 +57,9 @@ public class TriangleTreeTests extends ESTestCase { public void testDimensionalShapeType() throws IOException { GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); assertDimensionalShapeType(randomPoint(false), DimensionalShapeType.POINT); - assertDimensionalShapeType(randomMultiPoint(false), DimensionalShapeType.MULTIPOINT); - assertDimensionalShapeType(randomLine(false), DimensionalShapeType.LINESTRING); - assertDimensionalShapeType(randomMultiLine(false), DimensionalShapeType.MULTILINESTRING); + assertDimensionalShapeType(randomMultiPoint(false), DimensionalShapeType.POINT); + assertDimensionalShapeType(randomLine(false), DimensionalShapeType.LINE); + assertDimensionalShapeType(randomMultiLine(false), DimensionalShapeType.LINE); Geometry randoPoly = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { try { Geometry newGeo = indexer.prepareForIndexing(g); @@ -69,20 +69,20 @@ public void testDimensionalShapeType() throws IOException { } }, () -> randomPolygon(false))); assertDimensionalShapeType(randoPoly, DimensionalShapeType.POLYGON); - assertDimensionalShapeType(indexer.prepareForIndexing(randomMultiPolygon(false)), DimensionalShapeType.MULTIPOLYGON); + assertDimensionalShapeType(indexer.prepareForIndexing(randomMultiPolygon(false)), DimensionalShapeType.POLYGON); assertDimensionalShapeType(randomRectangle(), DimensionalShapeType.POLYGON); assertDimensionalShapeType(randomFrom( new GeometryCollection<>(List.of(randomPoint(false))), new GeometryCollection<>(List.of(randomMultiPoint(false))), new GeometryCollection<>(Collections.singletonList( new GeometryCollection<>(List.of(randomPoint(false), randomMultiPoint(false)))))) - , DimensionalShapeType.GEOMETRYCOLLECTION_POINTS); + , DimensionalShapeType.POINT); assertDimensionalShapeType(randomFrom( new GeometryCollection<>(List.of(randomPoint(false), randomLine(false))), new GeometryCollection<>(List.of(randomMultiPoint(false), randomMultiLine(false))), new GeometryCollection<>(Collections.singletonList( new GeometryCollection<>(List.of(randomPoint(false), randomLine(false)))))) - , DimensionalShapeType.GEOMETRYCOLLECTION_LINES); + , DimensionalShapeType.LINE); assertDimensionalShapeType(randomFrom( new GeometryCollection<>(List.of(randomPoint(false), indexer.prepareForIndexing(randomLine(false)), indexer.prepareForIndexing(randomPolygon(false)))), @@ -90,7 +90,7 @@ public void testDimensionalShapeType() throws IOException { new GeometryCollection<>(Collections.singletonList( new GeometryCollection<>(List.of(indexer.prepareForIndexing(randomLine(false)), indexer.prepareForIndexing(randomPolygon(false))))))) - , DimensionalShapeType.GEOMETRYCOLLECTION_POLYGONS); + , DimensionalShapeType.POLYGON); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java index 892901fb9965d..57e56748d2f65 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorTests.java @@ -47,6 +47,8 @@ import java.util.List; import java.util.function.Function; +import static org.elasticsearch.common.geo.DimensionalShapeType.POINT; + public class GeoCentroidAggregatorTests extends AggregatorTestCase { private static final double GEOHASH_TOLERANCE = 1E-6D; @@ -176,7 +178,7 @@ public void testMultiValuedGeoPointField() throws Exception { public void testGeoShapeField() throws Exception { int numDocs = scaledRandomIntBetween(64, 256); List geometries = new ArrayList<>(); - DimensionalShapeType targetShapeType = null; + DimensionalShapeType targetShapeType = POINT; GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); for (int i = 0; i < numDocs; i++) { Function geometryGenerator = ESTestCase.randomFrom( @@ -194,7 +196,10 @@ public void testGeoShapeField() throws Exception { } catch (InvalidShapeException e) { // do not include geometry } - targetShapeType = DimensionalShapeType.max(targetShapeType, DimensionalShapeType.forGeometry(geometry)); + // find dimensional-shape-type of geometry + CentroidCalculator centroidCalculator = new CentroidCalculator(geometry); + DimensionalShapeType geometryShapeType = centroidCalculator.getDimensionalShapeType(); + targetShapeType = targetShapeType.compareTo(geometryShapeType) >= 0 ? targetShapeType : geometryShapeType; } try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { @@ -206,7 +211,7 @@ public void testGeoShapeField() throws Exception { CentroidCalculator calculator = new CentroidCalculator(geometry); document.add(new BinaryGeoShapeDocValuesField("field", GeoTestUtils.toDecodedTriangles(geometry), calculator)); w.addDocument(document); - if (DimensionalShapeType.COMPARATOR.compare(targetShapeType, calculator.getDimensionalShapeType()) == 0) { + if (targetShapeType.compareTo(calculator.getDimensionalShapeType()) == 0) { double weight = calculator.sumWeight(); compensatedSumLat.add(weight * calculator.getY()); compensatedSumLon.add(weight * calculator.getX()); From 4c410f0cdb1d8627702aecdfabad569791527da4 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 24 Feb 2020 09:58:53 -0800 Subject: [PATCH 59/62] add `doc_values` mapping option to geo_shape field mapping (#47519) This PR adds support for the `doc_values` field mapping parameter. `true` and `false` supported by the GeoShapeFieldMapper, only `false` is supported by the LegacyGeoShapeFieldMapper. relates #37206 --- .../mapping/types/geo-shape.asciidoc | 5 ++ .../mapper/AbstractGeometryFieldMapper.java | 40 +++++++++--- .../index/mapper/GeoShapeFieldMapper.java | 16 ++++- .../mapper/LegacyGeoShapeFieldMapper.java | 19 ++++-- .../mapper/GeoShapeFieldMapperTests.java | 60 ++++++++++++++++- .../LegacyGeoShapeFieldMapperTests.java | 64 +++++++++++++++++++ .../index/mapper/ShapeFieldMapper.java | 16 ++++- .../index/mapper/ShapeFieldMapperTests.java | 54 ++++++++++++++++ 8 files changed, 253 insertions(+), 21 deletions(-) diff --git a/docs/reference/mapping/types/geo-shape.asciidoc b/docs/reference/mapping/types/geo-shape.asciidoc index 274970e0668a0..7e2c0241d7d16 100644 --- a/docs/reference/mapping/types/geo-shape.asciidoc +++ b/docs/reference/mapping/types/geo-shape.asciidoc @@ -114,6 +114,11 @@ and reject the whole document. |`coerce` |If `true` unclosed linear rings in polygons will be automatically closed. | `false` +|`doc_values` |Should the field be stored on disk in a column-stride fashion, so that it + can later be used for sorting, aggregations, or scripting? Accepts `true` + (default) or `false`. +| `true` for BKD-backed geo_shape, `false` for prefix tree indexing strategy + |======================================================================= diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index fddadb513a0ed..7ef03a8632aa1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -70,6 +70,7 @@ public static class Defaults { public static final Explicit COERCE = new Explicit<>(false, false); public static final Explicit IGNORE_MALFORMED = new Explicit<>(false, false); public static final Explicit IGNORE_Z_VALUE = new Explicit<>(true, false); + public static final Explicit DOC_VALUES = new Explicit<>(false, false); } @@ -122,15 +123,6 @@ public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFi super(name, fieldType, defaultFieldType); } - public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType, - boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ) { - super(name, fieldType, defaultFieldType); - this.coerce = coerce; - this.ignoreMalformed = ignoreMalformed; - this.orientation = orientation; - this.ignoreZValue = ignoreZ; - } - public Builder coerce(boolean coerce) { this.coerce = coerce; return this; @@ -190,6 +182,15 @@ public Builder ignoreZValue(final boolean ignoreZValue) { return this; } + protected Explicit docValues() { + if (docValuesSet && fieldType.hasDocValues()) { + return new Explicit<>(true, true); + } else if (docValuesSet) { + return new Explicit<>(false, true); + } + return Defaults.DOC_VALUES; + } + @Override protected void setupFieldType(BuilderContext context) { super.setupFieldType(context); @@ -251,6 +252,9 @@ public Mapper.Builder parse(String name, Map node, ParserContext XContentMapValues.nodeBooleanValue(fieldNode, name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName())); iterator.remove(); + } else if (TypeParsers.DOC_VALUES.equals(fieldName)) { + params.put(TypeParsers.DOC_VALUES, XContentMapValues.nodeBooleanValue(fieldNode, name + "." + TypeParsers.DOC_VALUES)); + iterator.remove(); } } if (parsedDeprecatedParameters == false) { @@ -258,6 +262,10 @@ public Mapper.Builder parse(String name, Map node, ParserContext } Builder builder = newBuilder(name, params); + if (params.containsKey(TypeParsers.DOC_VALUES)) { + builder.docValues((Boolean) params.get(TypeParsers.DOC_VALUES)); + } + if (params.containsKey(Names.COERCE.getPreferredName())) { builder.coerce((Boolean)params.get(Names.COERCE.getPreferredName())); } @@ -358,15 +366,17 @@ public QueryProcessor geometryQueryBuilder() { protected Explicit coerce; protected Explicit ignoreMalformed; protected Explicit ignoreZValue; + protected Explicit docValues; protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, - Explicit ignoreZValue, Settings indexSettings, + Explicit ignoreZValue, Explicit docValues, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); this.coerce = coerce; this.ignoreMalformed = ignoreMalformed; this.ignoreZValue = ignoreZValue; + this.docValues = docValues; } @Override @@ -382,6 +392,9 @@ protected void doMerge(Mapper mergeWith) { if (gsfm.ignoreZValue.explicit()) { this.ignoreZValue = gsfm.ignoreZValue; } + if (gsfm.docValues.explicit()) { + this.docValues = gsfm.docValues; + } } @Override @@ -405,6 +418,9 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par if (includeDefaults || ignoreZValue.explicit()) { builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value()); } + if (includeDefaults || docValues.explicit()) { + builder.field(TypeParsers.DOC_VALUES, docValues.value()); + } } public Explicit coerce() { @@ -419,6 +435,10 @@ public Explicit ignoreZValue() { return ignoreZValue; } + public Explicit docValues() { + return docValues; + } + public Orientation orientation() { return ((AbstractGeometryFieldType)fieldType).orientation(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java index 6f9cff5bd82d6..320c41284caad 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java @@ -67,7 +67,8 @@ public Builder(String name) { public GeoShapeFieldMapper build(BuilderContext context) { setupFieldType(context); return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), - ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + ignoreZValue(), docValues(), context.indexSettings(), + multiFieldsBuilder.build(this, context), copyTo); } @Override @@ -75,6 +76,15 @@ public boolean defaultDocValues(Version indexCreated) { return Version.V_8_0_0.onOrBefore(indexCreated); } + protected Explicit docValues() { + if (docValuesSet && fieldType.hasDocValues()) { + return new Explicit<>(true, true); + } else if (docValuesSet) { + return new Explicit<>(false, true); + } + return new Explicit<>(fieldType.hasDocValues(), false); + } + protected void setupFieldType(BuilderContext context) { super.setupFieldType(context); @@ -131,9 +141,9 @@ public String typeName() { public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, - Explicit ignoreZValue, Settings indexSettings, + Explicit ignoreZValue, Explicit docValues, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, indexSettings, multiFields, copyTo); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 6fea1efaedafe..3fc5ac28c7b49 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -197,6 +197,16 @@ public GeoShapeFieldType fieldType() { return (GeoShapeFieldType)fieldType; } + public Builder docValues(boolean hasDocValues) { + super.docValues(hasDocValues); + if (hasDocValues) { + throw new ElasticsearchParseException("geo_shape field [" + name + + "] indexed using prefix-trees do not support doc_values"); + } + // doc-values already set to `false` + return this; + } + private void setupFieldTypeDeprecatedParameters(BuilderContext context) { GeoShapeFieldType ft = fieldType(); if (deprecatedParameters.strategy != null) { @@ -292,7 +302,7 @@ public LegacyGeoShapeFieldMapper build(BuilderContext context) { setupFieldType(context); return new LegacyGeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), - coerce(context), orientation(), ignoreZValue(), context.indexSettings(), + coerce(context), orientation(), ignoreZValue(), docValues(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); } } @@ -318,6 +328,7 @@ public GeoShapeFieldType() { setStored(false); setStoreTermVectors(false); setOmitNorms(true); + setHasDocValues(false); } protected GeoShapeFieldType(GeoShapeFieldType ref) { @@ -470,10 +481,10 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) { public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, Explicit orientation, - Explicit ignoreZValue, Settings indexSettings, + Explicit ignoreZValue, Explicit docValues, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, - multiFields, copyTo); + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, + indexSettings, multiFields, copyTo); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 280668c1c6dbc..a2c20762097b2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -60,7 +60,9 @@ public void testDefaultConfiguration() throws IOException { GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value())); - assertTrue(geoShapeFieldMapper.fieldType.hasDocValues()); + assertFalse(geoShapeFieldMapper.docValues().explicit()); + assertTrue(geoShapeFieldMapper.docValues().value()); + assertTrue(geoShapeFieldMapper.fieldType().hasDocValues()); } /** @@ -214,6 +216,45 @@ public void testIgnoreMalformedParsing() throws IOException { assertThat(ignoreMalformed.value(), equalTo(false)); } + /** + * Test that doc_values parameter correctly parses + */ + public void testDocValues() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("doc_values", true) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().explicit()); + assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().value()); + boolean hasDocValues = ((GeoShapeFieldMapper)fieldMapper).fieldType().hasDocValues(); + assertTrue(hasDocValues); + + // explicit false doc_values + mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("doc_values", "false") + .endObject().endObject() + .endObject().endObject()); + + defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().explicit()); + assertFalse(((GeoShapeFieldMapper)fieldMapper).docValues().value()); + hasDocValues = ((GeoShapeFieldMapper)fieldMapper).fieldType().hasDocValues(); + assertFalse(hasDocValues); + } private void assertFieldWarnings(String... fieldNames) { String[] warnings = new String[fieldNames.length]; @@ -283,9 +324,26 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"orientation\":\"" + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":true")); } } + public void testSerializeDocValues() throws IOException { + boolean docValues = randomBoolean(); + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("doc_values", docValues) + .endObject().endObject() + .endObject().endObject()); + DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((GeoShapeFieldMapper) mapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":" + docValues)); + } + public String toXContentString(GeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); ToXContent.Params params; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java index aaabf3f9edbf0..2b17ba8a55fa8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -83,6 +83,10 @@ public void testDefaultConfiguration() throws IOException { equalTo(LegacyGeoShapeFieldMapper.DeprecatedParameters.Defaults.DISTANCE_ERROR_PCT)); assertThat(geoShapeFieldMapper.fieldType().orientation(), equalTo(LegacyGeoShapeFieldMapper.Defaults.ORIENTATION.value())); + assertThat(geoShapeFieldMapper.docValues(), + equalTo(LegacyGeoShapeFieldMapper.Defaults.DOC_VALUES)); + assertThat(geoShapeFieldMapper.fieldType().hasDocValues(), + equalTo(LegacyGeoShapeFieldMapper.Defaults.DOC_VALUES.value())); assertFieldWarnings("strategy"); } @@ -598,6 +602,7 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); assertTrue(serialized, serialized.contains("\"tree_levels\":21")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -610,6 +615,7 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"precision\":\"50.0m\"")); assertTrue(serialized, serialized.contains("\"tree_levels\":9")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -623,6 +629,7 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertFalse(serialized, serialized.contains("\"precision\":")); assertTrue(serialized, serialized.contains("\"tree_levels\":6")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -636,6 +643,7 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); assertFalse(serialized, serialized.contains("\"tree_levels\":")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") @@ -650,10 +658,29 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((LegacyGeoShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"precision\":\"6.0m\"")); assertTrue(serialized, serialized.contains("\"tree_levels\":5")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } assertFieldWarnings("tree", "tree_levels", "precision"); } + public void testSerializeDocValues() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("doc_values", false) + .endObject().endObject() + .endObject().endObject()); + DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((LegacyGeoShapeFieldMapper) mapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); + + assertFieldWarnings("tree"); + } + public void testPointsOnlyDefaultsWithTermStrategy() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location") @@ -702,6 +729,43 @@ public void testPointsOnlyFalseWithTermStrategy() throws Exception { assertFieldWarnings("tree", "precision", "strategy", "points_only"); } + /** + * Test that doc_values parameter correctly parses + */ + public void testDocValues() throws IOException { + String trueMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("doc_values", true) + .endObject().endObject() + .endObject().endObject()); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(trueMapping))); + assertThat(e.getMessage(), equalTo("geo_shape field [location] indexed using prefix-trees do not support doc_values")); + + // explicit false doc_values + String falseMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .field("tree", "quadtree") + .field("doc_values", "false") + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(falseMapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(LegacyGeoShapeFieldMapper.class)); + + assertTrue(((LegacyGeoShapeFieldMapper) fieldMapper).docValues().explicit()); + assertFalse(((LegacyGeoShapeFieldMapper) fieldMapper).docValues().value()); + assertFalse(((LegacyGeoShapeFieldMapper) fieldMapper).fieldType().hasDocValues()); + + assertFieldWarnings("tree"); + } + public void testDisallowExpensiveQueries() throws IOException { QueryShardContext queryShardContext = mock(QueryShardContext.class); when(queryShardContext.allowExpensiveQueries()).thenReturn(false); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java index 472caa6e8ed86..7768e3ed0804a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapper.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.spatial.index.mapper; import org.apache.lucene.document.XYShape; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.geo.builders.ShapeBuilder; @@ -53,7 +54,16 @@ public Builder(String name) { public ShapeFieldMapper build(BuilderContext context) { setupFieldType(context); return new ShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context), - ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + ignoreZValue(), docValues(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + } + + public ShapeFieldMapper.Builder docValues(boolean hasDocValues) { + super.docValues(hasDocValues); + if (hasDocValues) { + throw new ElasticsearchParseException("field [" + name + "] of type [" + fieldType().typeName() + + "] does not support doc-values"); + } + return this; } @Override @@ -116,9 +126,9 @@ protected Indexer geometryIndexer() { public ShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, Explicit ignoreMalformed, Explicit coerce, - Explicit ignoreZValue, Settings indexSettings, + Explicit ignoreZValue, Explicit docValues, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { - super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings, + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, indexSettings, multiFields, copyTo); } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 809f9d621395e..d8012a6ea2f42 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.spatial.index.mapper; +import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -54,6 +55,9 @@ public void testDefaultConfiguration() throws IOException { ShapeFieldMapper shapeFieldMapper = (ShapeFieldMapper) fieldMapper; assertThat(shapeFieldMapper.fieldType().orientation(), equalTo(ShapeFieldMapper.Defaults.ORIENTATION.value())); + assertFalse(shapeFieldMapper.docValues().value()); + assertFalse(shapeFieldMapper.docValues().explicit()); + assertFalse(shapeFieldMapper.fieldType().hasDocValues()); } /** @@ -96,6 +100,40 @@ public void testOrientationParsing() throws IOException { assertThat(orientation, equalTo(ShapeBuilder.Orientation.CCW)); } + /** + * Test that doc_values parameter correctly parses + */ + public void testDocValues() throws IOException { + String trueMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "shape") + .field("doc_values", true) + .endObject().endObject() + .endObject().endObject()); + + ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, + () -> createIndex("test").mapperService().documentMapperParser().parse("type1", new CompressedXContent(trueMapping))); + assertThat(e.getMessage(), equalTo("field [location] of type [shape] does not support doc-values")); + + // explicit false doc_values + String falseMapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "shape") + .field("doc_values", false) + .endObject().endObject() + .endObject().endObject()); + + DocumentMapper defaultMapper = createIndex("test2").mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(falseMapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(ShapeFieldMapper.class)); + + // since shape field has no doc-values, this field is ignored + assertTrue(((ShapeFieldMapper)fieldMapper).docValues().explicit()); + assertFalse(((ShapeFieldMapper)fieldMapper).docValues().value()); + assertFalse(((ShapeFieldMapper)fieldMapper).fieldType().hasDocValues()); + } + /** * Test that coerce parameter correctly parses */ @@ -276,9 +314,25 @@ public void testSerializeDefaults() throws Exception { String serialized = toXContentString((ShapeFieldMapper) defaultMapper.mappers().getMapper("location")); assertTrue(serialized, serialized.contains("\"orientation\":\"" + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); } } + public void testSerializeDocValues() throws IOException { + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "shape") + .field("doc_values", false) + .endObject().endObject() + .endObject().endObject()); + DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping)); + String serialized = toXContentString((ShapeFieldMapper) mapper.mappers().getMapper("location")); + assertTrue(serialized, serialized.contains("\"orientation\":\"" + + AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\"")); + assertTrue(serialized, serialized.contains("\"doc_values\":false")); + } + public String toXContentString(ShapeFieldMapper mapper, boolean includeDefaults) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); ToXContent.Params params; From d960712763b0d890555ba1832fe3842a108f460b Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Wed, 26 Feb 2020 08:58:38 -0800 Subject: [PATCH 60/62] ignore infinite centroid values in CentroidCalculator (#52782) there are times where small triangle areas within a polygon have really small areas 1e-11, while the whole polygon's area is zero. This results in an infinite valuation of the centroid point representing that triangle. This commit ignores the addition of such values Addresses #52774 --- .../common/geo/CentroidCalculator.java | 25 +++++++------- .../common/geo/CentroidCalculatorTests.java | 33 +++++++++++++++++++ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java index 466384c849b2f..5eb84f64aa6bd 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java +++ b/server/src/main/java/org/elasticsearch/common/geo/CentroidCalculator.java @@ -67,17 +67,20 @@ public CentroidCalculator(Geometry geometry) { * @param weight the associated weight of the coordinate */ private void addCoordinate(double x, double y, double weight, DimensionalShapeType dimensionalShapeType) { - if (this.dimensionalShapeType == null || this.dimensionalShapeType == dimensionalShapeType) { - compSumX.add(x * weight); - compSumY.add(y * weight); - compSumWeight.add(weight); - this.dimensionalShapeType = dimensionalShapeType; - } else if (dimensionalShapeType.compareTo(this.dimensionalShapeType) > 0) { - // reset counters - compSumX.reset(x * weight, 0); - compSumY.reset(y * weight, 0); - compSumWeight.reset(weight, 0); - this.dimensionalShapeType = dimensionalShapeType; + // x and y can be infinite due to really small areas and rounding problems + if (Double.isFinite(x) && Double.isFinite(y)) { + if (this.dimensionalShapeType == null || this.dimensionalShapeType == dimensionalShapeType) { + compSumX.add(x * weight); + compSumY.add(y * weight); + compSumWeight.add(weight); + this.dimensionalShapeType = dimensionalShapeType; + } else if (dimensionalShapeType.compareTo(this.dimensionalShapeType) > 0) { + // reset counters + compSumX.reset(x * weight, 0); + compSumY.reset(y * weight, 0); + compSumWeight.reset(weight, 0); + this.dimensionalShapeType = dimensionalShapeType; + } } } diff --git a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java index 5a5fc222debd5..9cf05e08b7d91 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/CentroidCalculatorTests.java @@ -28,6 +28,8 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Polygon; import org.elasticsearch.geometry.ShapeType; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -38,6 +40,7 @@ import static org.elasticsearch.common.geo.DimensionalShapeType.POINT; import static org.elasticsearch.common.geo.DimensionalShapeType.POLYGON; import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; public class CentroidCalculatorTests extends ESTestCase { @@ -52,6 +55,36 @@ public void testPoint() { assertThat(calculator.getDimensionalShapeType(), equalTo(POINT)); } + public void testPolygonWithSmallTrianglesOfZeroWeight() throws Exception { + Geometry geometry = new WellKnownText(false, new GeographyValidator(true)) + .fromWKT("POLYGON((-4.385064 55.2259599,-4.385056 55.2259224,-4.3850466 55.2258994,-4.3849755 55.2258574," + + "-4.3849339 55.2258589,-4.3847033 55.2258742,-4.3846805 55.2258818,-4.3846282 55.2259132,-4.3846215 55.2259247," + + "-4.3846121 55.2259683,-4.3846147 55.2259798,-4.3846369 55.2260157,-4.3846472 55.2260241," + + "-4.3846697 55.2260409,-4.3846952 55.2260562,-4.384765 55.22608,-4.3848199 55.2260861,-4.3848481 55.2260845," + + "-4.3849245 55.2260761,-4.3849393 55.22607,-4.3849996 55.2260432,-4.3850131 55.2260364,-4.3850426 55.2259989," + + "-4.385064 55.2259599),(-4.3850104 55.2259583,-4.385005 55.2259752,-4.384997 55.2259892,-4.3849339 55.2259981," + + "-4.3849272 55.2259308,-4.3850016 55.2259262,-4.385005 55.2259377,-4.3850104 55.2259583)," + + "(-4.3849996 55.2259193,-4.3847502 55.2259331,-4.3847548 55.2258921,-4.3848012 55.2258895," + + "-4.3849219 55.2258811,-4.3849514 55.2258818,-4.3849728 55.2258933,-4.3849996 55.2259193)," + + "(-4.3849917 55.2259984,-4.3849849 55.2260103,-4.3849771 55.2260192,-4.3849701 55.2260019,-4.3849917 55.2259984)," + + "(-4.3846608 55.2259374,-4.384663 55.2259316,-4.3846711 55.2259201,-4.3846992 55.225904," + + "-4.384718 55.2258941,-4.3847434 55.2258927,-4.3847314 55.2259407,-4.3849098 55.2259316,-4.3849098 55.2259492," + + "-4.3848843 55.2259515,-4.3849017 55.2260119,-4.3849567 55.226005,-4.3849701 55.2260272,-4.3849299 55.2260486," + + "-4.3849192 55.2260295,-4.384883 55.2260188,-4.3848776 55.2260119,-4.3848441 55.2260149,-4.3848441 55.2260226," + + "-4.3847864 55.2260241,-4.384722 55.2259652,-4.3847053 55.2259706,-4.384683 55.225954,-4.3846608 55.2259374)," + + "(-4.3846541 55.2259549,-4.384698 55.2259883,-4.3847173 55.2259828,-4.3847743 55.2260333,-4.3847891 55.2260356," + + "-4.3848146 55.226031,-4.3848199 55.2260409,-4.3848387 55.2260417,-4.3848494 55.2260593,-4.3848092 55.2260616," + + "-4.3847623 55.2260539,-4.3847341 55.2260432,-4.3847046 55.2260279,-4.3846738 55.2260062,-4.3846496 55.2259844," + + "-4.3846429 55.2259737,-4.3846523 55.2259714,-4.384651 55.2259629,-4.3846541 55.2259549)," + + "(-4.3846608 55.2259374,-4.3846559 55.2259502,-4.3846541 55.2259549,-4.3846608 55.2259374))"); + CentroidCalculator calculator = new CentroidCalculator(geometry); + assertThat(calculator.getX(), closeTo( -4.3848, 1e-4)); + assertThat(calculator.getY(), closeTo(55.22595, 1e-4)); + assertThat(calculator.sumWeight(), closeTo(0, 1e-5)); + assertThat(calculator.getDimensionalShapeType(), equalTo(POLYGON)); + } + + public void testLine() { double[] y = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; double[] x = new double[] { 10, 20, 30, 40, 50, 60, 70, 80, 90, 100 }; From b136fa70791891a232840fb034413f57a2bc1d9c Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 2 Mar 2020 08:39:24 -0800 Subject: [PATCH 61/62] Add bounds support for geogrid agg on shapes (#51973) This PR cleans up some aspects of GeoShapeCellValues to support the specialization of bounded geo_shape geo-grid aggregations. This refactor reverts some of the BoundedCellValues constructs. Instead, BoundedGeoTileGridTiler and BoundedGeoHashGridTiler are introduced. As part of this change, the definition/semantics of geo_grid aggs with bounds on geo_point are modified to match the same behavior as geo_shapes, where it is the tile of the point that must intersect the bounds in order for the point to be accounted for --- .../common/geo/TriangleTreeReader.java | 1 - .../index/fielddata/MultiGeoValues.java | 1 - .../GeoTileGridValuesSourceBuilder.java | 12 +- .../geogrid/BoundedGeoHashGridTiler.java | 108 ++++++ .../geogrid/BoundedGeoPointCellValues.java | 48 --- .../geogrid/BoundedGeoTileGridTiler.java | 107 ++++++ .../bucket/geogrid/CellIdSource.java | 18 +- .../bucket/geogrid/CellValues.java | 13 + .../bucket/geogrid/GeoGridTiler.java | 255 ++------------ .../geogrid/GeoHashGridAggregatorFactory.java | 8 +- .../bucket/geogrid/GeoHashGridTiler.java | 133 ++++++++ ...ellValues.java => GeoPointCellValues.java} | 11 +- .../bucket/geogrid/GeoShapeCellValues.java | 41 +-- .../geogrid/GeoTileGridAggregatorFactory.java | 8 +- .../bucket/geogrid/GeoTileGridTiler.java | 166 ++++++++++ .../bucket/geogrid/GeoTileUtils.java | 33 +- .../metrics/InternalGeoBounds.java | 3 +- .../common/geo/TriangleTreeTests.java | 16 +- .../geogrid/GeoGridAggregatorTestCase.java | 176 +++++++--- .../bucket/geogrid/GeoGridTilerTests.java | 310 ++++++++++++++---- .../geogrid/GeoHashGridAggregatorTests.java | 15 + .../geogrid/GeoTileGridAggregatorTests.java | 15 + 22 files changed, 1020 insertions(+), 478 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java delete mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoPointCellValues.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTiler.java rename server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/{UnboundedGeoPointCellValues.java => GeoPointCellValues.java} (71%) create mode 100644 server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTiler.java diff --git a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java index cecbf227a3975..b6b06e5088162 100644 --- a/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java +++ b/server/src/main/java/org/elasticsearch/common/geo/TriangleTreeReader.java @@ -322,7 +322,6 @@ private GeoRelation relateTriangle(int aX, int aY, boolean ab, int bX, int bY, b int tMinY = StrictMath.min(StrictMath.min(aY, bY), cY); int tMaxY = StrictMath.max(StrictMath.max(aY, bY), cY); - // 1. check bounding boxes are disjoint, where north and east boundaries are not considered as crossing if (tMaxX <= minX || tMinX > maxX || tMinY > maxY || tMaxY <= minY) { return GeoRelation.QUERY_DISJOINT; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java index ea973fa91fa5f..e5bb77b4b20ca 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/MultiGeoValues.java @@ -316,6 +316,5 @@ public double minX() { public double maxX() { return Math.max(negRight, posRight); } - } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java index e06df02518e83..b47ae3029bffd 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/composite/GeoTileGridValuesSourceBuilder.java @@ -31,9 +31,11 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler; import org.elasticsearch.search.aggregations.bucket.geogrid.CellIdSource; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridTiler; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridTiler; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.search.aggregations.support.ValueType; import org.elasticsearch.search.aggregations.support.ValuesSource; @@ -138,7 +140,15 @@ protected CompositeValuesSourceConfig innerBuild(QueryShardContext queryShardCon ValuesSource.Geo geoValue = (ValuesSource.Geo) orig; // is specified in the builder. final MappedFieldType fieldType = config.fieldContext() != null ? config.fieldContext().fieldType() : null; - CellIdSource cellIdSource = new CellIdSource(geoValue, precision, geoBoundingBox, GeoGridTiler.GeoTileGridTiler.INSTANCE); + + final GeoGridTiler tiler; + if (geoBoundingBox.isUnbounded()) { + tiler = new GeoTileGridTiler(); + } else { + tiler = new BoundedGeoTileGridTiler(geoBoundingBox); + } + + CellIdSource cellIdSource = new CellIdSource(geoValue, precision, tiler); return new CompositeValuesSourceConfig(name, fieldType, cellIdSource, DocValueFormat.GEOTILE, order(), missingBucket(), script() != null); } else { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java new file mode 100644 index 0000000000000..0636de647a22c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoHashGridTiler.java @@ -0,0 +1,108 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +public class BoundedGeoHashGridTiler extends GeoHashGridTiler { + private final double boundsTop; + private final double boundsBottom; + private final double boundsWestLeft; + private final double boundsWestRight; + private final double boundsEastLeft; + private final double boundsEastRight; + private final boolean crossesDateline; + + BoundedGeoHashGridTiler(GeoBoundingBox geoBoundingBox) { + // split geoBoundingBox into west and east boxes + boundsTop = geoBoundingBox.top(); + boundsBottom = geoBoundingBox.bottom(); + if (geoBoundingBox.right() < geoBoundingBox.left()) { + boundsWestLeft = -180; + boundsWestRight = geoBoundingBox.right(); + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = geoBoundingBox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + } + + @Override + public int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + long hash = encode(x, y, precision); + if (cellIntersectsGeoBoundingBox(Geohash.toBoundingBox(Geohash.stringEncode(hash)))) { + values[valuesIdx] = hash; + return valuesIdx + 1; + } + return valuesIdx; + } + + boolean cellIntersectsGeoBoundingBox(Rectangle rectangle) { + return (boundsTop >= rectangle.getMinY() && boundsBottom <= rectangle.getMaxY() + && (boundsEastLeft <= rectangle.getMaxX() && boundsEastRight >= rectangle.getMinX() + || (crossesDateline && boundsWestLeft <= rectangle.getMaxX() && boundsWestRight >= rectangle.getMinX()))); + } + + @Override + protected int setValue(CellValues docValues, MultiGeoValues.GeoValue geoValue, MultiGeoValues.BoundingBox bounds, int precision) { + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + GeoRelation relation = relateTile(geoValue, hash); + if (relation != GeoRelation.QUERY_DISJOINT) { + docValues.resizeCell(1); + docValues.add(0, Geohash.longEncode(hash)); + return 1; + } + return 0; + } + + @Override + protected GeoRelation relateTile(MultiGeoValues.GeoValue geoValue, String hash) { + Rectangle rectangle = Geohash.toBoundingBox(hash); + if (cellIntersectsGeoBoundingBox(rectangle)) { + return geoValue.relate(rectangle); + } else { + return GeoRelation.QUERY_DISJOINT; + } + } + + @Override + protected int setValuesForFullyContainedTile(String hash, CellValues values, + int valuesIndex, int targetPrecision) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + if (hashes[i].length() == targetPrecision ) { + if (cellIntersectsGeoBoundingBox(Geohash.toBoundingBox(hashes[i]))) { + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } + } else { + valuesIndex = setValuesForFullyContainedTile(hashes[i], values, valuesIndex, targetPrecision); + } + } + return valuesIndex; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoPointCellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoPointCellValues.java deleted file mode 100644 index 493a9aa729678..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoPointCellValues.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.search.aggregations.bucket.geogrid; - -import org.elasticsearch.common.geo.GeoBoundingBox; -import org.elasticsearch.index.fielddata.MultiGeoValues; - -/** - * Class representing {@link CellValues} whose values are filtered - * according to whether they are within the specified {@link GeoBoundingBox}. - * - * The specified bounding box is assumed to be bounded. - */ -class BoundedGeoPointCellValues extends CellValues { - - private final GeoBoundingBox geoBoundingBox; - - protected BoundedGeoPointCellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler, GeoBoundingBox geoBoundingBox) { - super(geoValues, precision, tiler); - this.geoBoundingBox = geoBoundingBox; - } - - - @Override - int advanceValue(MultiGeoValues.GeoValue target, int valuesIdx) { - if (geoBoundingBox.pointInBounds(target.lon(), target.lat())) { - values[valuesIdx] = tiler.encode(target.lon(), target.lat(), precision); - return valuesIdx + 1; - } - return valuesIdx; - } -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java new file mode 100644 index 0000000000000..89b6b69c41ba7 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/BoundedGeoTileGridTiler.java @@ -0,0 +1,107 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +public class BoundedGeoTileGridTiler extends GeoTileGridTiler { + private final double boundsTop; + private final double boundsBottom; + private final double boundsWestLeft; + private final double boundsWestRight; + private final double boundsEastLeft; + private final double boundsEastRight; + private final boolean crossesDateline; + + public BoundedGeoTileGridTiler(GeoBoundingBox geoBoundingBox) { + // split geoBoundingBox into west and east boxes + boundsTop = geoBoundingBox.top(); + boundsBottom = geoBoundingBox.bottom(); + if (geoBoundingBox.right() < geoBoundingBox.left()) { + boundsWestLeft = -180; + boundsWestRight = geoBoundingBox.right(); + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = geoBoundingBox.left(); + boundsEastRight = geoBoundingBox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + } + + public int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + long hash = encode(x, y, precision); + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(hash))) { + values[valuesIdx] = hash; + return valuesIdx + 1; + } + return valuesIdx; + } + + boolean cellIntersectsGeoBoundingBox(Rectangle rectangle) { + return (boundsTop >= rectangle.getMinY() && boundsBottom <= rectangle.getMaxY() + && (boundsEastLeft <= rectangle.getMaxX() && boundsEastRight >= rectangle.getMinX() + || (crossesDateline && boundsWestLeft <= rectangle.getMaxX() && boundsWestRight >= rectangle.getMinX()))); + } + + @Override + public GeoRelation relateTile(MultiGeoValues.GeoValue geoValue, int xTile, int yTile, int precision) { + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + if (cellIntersectsGeoBoundingBox(rectangle)) { + return geoValue.relate(rectangle); + } + return GeoRelation.QUERY_DISJOINT; + } + + @Override + protected int setValue(CellValues docValues, MultiGeoValues.GeoValue geoValue, int xTile, int yTile, int precision) { + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(xTile, yTile, precision))) { + docValues.resizeCell(1); + docValues.add(0, GeoTileUtils.longEncodeTiles(precision, xTile, yTile)); + return 1; + } + return 0; + } + + @Override + protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, CellValues values, int valuesIndex, + int targetPrecision) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + if (zTile == targetPrecision) { + if (cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(nextX, nextY, zTile))) { + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } + } else { + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java index 8161aef98746c..8267bc69f8c52 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellIdSource.java @@ -21,7 +21,6 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.MultiGeoValues; -import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; @@ -36,13 +35,11 @@ public class CellIdSource extends ValuesSource.Numeric { private final ValuesSource.Geo valuesSource; private final int precision; private final GeoGridTiler encoder; - private final GeoBoundingBox geoBoundingBox; - public CellIdSource(ValuesSource.Geo valuesSource, int precision, GeoBoundingBox geoBoundingBox, GeoGridTiler encoder) { + public CellIdSource(ValuesSource.Geo valuesSource, int precision, GeoGridTiler encoder) { this.valuesSource = valuesSource; //different GeoPoints could map to the same or different hashing cells. this.precision = precision; - this.geoBoundingBox = geoBoundingBox; this.encoder = encoder; } @@ -65,19 +62,10 @@ public SortedNumericDocValues longValues(LeafReaderContext ctx) { ValuesSourceType vs = geoValues.valuesSourceType(); if (CoreValuesSourceType.GEOPOINT == vs) { // docValues are geo points - if (geoBoundingBox.isUnbounded()) { - return new UnboundedGeoPointCellValues(geoValues, precision, encoder); - } else { - return new BoundedGeoPointCellValues(geoValues, precision, encoder, geoBoundingBox); - } + return new GeoPointCellValues(geoValues, precision, encoder); } else if (CoreValuesSourceType.GEOSHAPE == vs || CoreValuesSourceType.GEO == vs) { // docValues are geo shapes - if (geoBoundingBox.isUnbounded()) { - return new GeoShapeCellValues(geoValues, precision, encoder); - } else { - // TODO(talevy): support unbounded - throw new IllegalArgumentException("bounded geogrid is not supported on geo_shape fields"); - } + return new GeoShapeCellValues(geoValues, precision, encoder); } else { throw new IllegalArgumentException("unsupported geo type"); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellValues.java index b53d02e81a2ef..6ebad166f513d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/CellValues.java @@ -56,6 +56,19 @@ public boolean advanceExact(int docId) throws IOException { } } + // for testing + protected long[] getValues() { + return values; + } + + protected void add(int idx, long value) { + values[idx] = value; + } + + void resizeCell(int newSize) { + resize(newSize); + } + /** * Sets the appropriate long-encoded value for target * in values. diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java index ebf7ad21d208c..1f98f9bdf3031 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTiler.java @@ -19,9 +19,6 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.GeoRelation; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.geometry.utils.Geohash; import org.elasticsearch.index.fielddata.MultiGeoValues; /** @@ -39,241 +36,31 @@ public interface GeoGridTiler { /** * - * @param docValues the array of long-encoded bucket keys to fill - * @param geoValue the input shape - * @param precision the tile zoom-level + * @param docValues the array of long-encoded bucket keys to fill + * @param geoValue the input shape + * @param precision the tile zoom-level * * @return the number of tiles the geoValue intersects */ - int setValues(GeoShapeCellValues docValues, MultiGeoValues.GeoValue geoValue, int precision); + int setValues(CellValues docValues, MultiGeoValues.GeoValue geoValue, int precision); - class GeoHashGridTiler implements GeoGridTiler { - public static final GeoHashGridTiler INSTANCE = new GeoHashGridTiler(); - private GeoHashGridTiler() {} - - @Override - public long encode(double x, double y, int precision) { - return Geohash.longEncode(x, y, precision); - } - - @Override - public int setValues(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, int precision) { - if (precision == 1) { - values.resizeCell(1); - values.add(0, Geohash.longEncode(0, 0, 0)); - } - - MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); - assert bounds.minX() <= bounds.maxX(); - long numLonCells = (long) ((bounds.maxX() - bounds.minX()) / Geohash.lonWidthInDegrees(precision)); - long numLatCells = (long) ((bounds.maxY() - bounds.minY()) / Geohash.latHeightInDegrees(precision)); - long count = (numLonCells + 1) * (numLatCells + 1); - if (count == 1) { - String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); - values.resizeCell(1); - values.add(0, Geohash.longEncode(hash)); - return 1; - } else if (count <= precision) { - return setValuesByBruteForceScan(values, geoValue, precision, bounds); - } else { - return setValuesByRasterization("", values, 0, precision, geoValue); - } - } - - protected int setValuesByBruteForceScan(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, - int precision, MultiGeoValues.BoundingBox bounds) { - // TODO: This way to discover cells inside of a bounding box seems not to work as expected. I can - // see that eventually we will be visiting twice the same cell which should not happen. - int idx = 0; - String min = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); - String max = Geohash.stringEncode(bounds.maxX(), bounds.maxY(), precision); - String minNeighborBelow = Geohash.getNeighbor(min, precision, 0, -1); - double minY = Geohash.decodeLatitude((minNeighborBelow == null) ? min : minNeighborBelow); - double minX = Geohash.decodeLongitude(min); - double maxY = Geohash.decodeLatitude(max); - double maxX = Geohash.decodeLongitude(max); - for (double i = minX; i <= maxX; i += Geohash.lonWidthInDegrees(precision)) { - for (double j = minY; j <= maxY; j += Geohash.latHeightInDegrees(precision)) { - Rectangle rectangle = Geohash.toBoundingBox(Geohash.stringEncode(i, j, precision)); - GeoRelation relation = geoValue.relate(rectangle); - if (relation != GeoRelation.QUERY_DISJOINT) { - values.resizeCell(idx + 1); - values.add(idx++, encode(i, j, precision)); - } - } - } - return idx; - } - - protected int setValuesByRasterization(String hash, GeoShapeCellValues values, int valuesIndex, - int targetPrecision, MultiGeoValues.GeoValue geoValue) { - String[] hashes = Geohash.getSubGeohashes(hash); - for (int i = 0; i < hashes.length; i++) { - Rectangle rectangle = Geohash.toBoundingBox(hashes[i]); - GeoRelation relation = geoValue.relate(rectangle); - if (relation == GeoRelation.QUERY_CROSSES) { - if (hashes[i].length() == targetPrecision) { - values.resizeCell(valuesIndex + 1); - values.add(valuesIndex++, Geohash.longEncode(hashes[i])); - } else { - valuesIndex = - setValuesByRasterization(hashes[i], values, valuesIndex, targetPrecision, geoValue); - } - } else if (relation == GeoRelation.QUERY_INSIDE) { - if (hashes[i].length() == targetPrecision) { - values.resizeCell(valuesIndex + 1); - values.add(valuesIndex++, Geohash.longEncode(hashes[i])); - } else { - values.resizeCell(valuesIndex + (int) Math.pow(32, targetPrecision - hash.length()) + 1); - valuesIndex = setValuesForFullyContainedTile(hashes[i],values, valuesIndex, targetPrecision); - } - } - } - return valuesIndex; - } - - private int setValuesForFullyContainedTile(String hash, GeoShapeCellValues values, - int valuesIndex, int targetPrecision) { - String[] hashes = Geohash.getSubGeohashes(hash); - for (int i = 0; i < hashes.length; i++) { - if (hashes[i].length() == targetPrecision) { - values.add(valuesIndex++, Geohash.longEncode(hashes[i])); - } else { - valuesIndex = setValuesForFullyContainedTile(hashes[i], values, valuesIndex, targetPrecision); - } - } - return valuesIndex; - } - } - - class GeoTileGridTiler implements GeoGridTiler { - public static final GeoTileGridTiler INSTANCE = new GeoTileGridTiler(); - - private GeoTileGridTiler() {} - - @Override - public long encode(double x, double y, int precision) { - return GeoTileUtils.longEncode(x, y, precision); - } - - /** - * Sets the values of the long[] underlying {@link GeoShapeCellValues}. - * - * If the shape resides between GeoTileUtils.NORMALIZED_LATITUDE_MASK and 90 or - * between GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK and -90 degree latitudes, then - * the shape is not accounted for since geo-tiles are only defined within those bounds. - * - * @param values the bucket values - * @param geoValue the input shape - * @param precision the tile zoom-level - * - * @return the number of tiles set by the shape - */ - @Override - public int setValues(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, int precision) { - MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); - assert bounds.minX() <= bounds.maxX(); - - if (precision == 0) { - values.resizeCell(1); - values.add(0, GeoTileUtils.longEncodeTiles(0, 0, 0)); - return 1; - } - - // geo tiles are not defined at the extreme latitudes due to them - // tiling the world as a square. - if ((bounds.top > GeoTileUtils.NORMALIZED_LATITUDE_MASK && bounds.bottom > GeoTileUtils.NORMALIZED_LATITUDE_MASK) - || (bounds.top < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK - && bounds.bottom < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK)) { - return 0; - } - - - final double tiles = 1 << precision; - int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); - int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); - int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); - int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); - int count = (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1); - if (count == 1) { - values.resizeCell(1); - values.add(0, GeoTileUtils.longEncodeTiles(precision, minXTile, minYTile)); - return 1; - } else if (count <= precision) { - return setValuesByBruteForceScan(values, geoValue, precision, minXTile, minYTile, maxXTile, maxYTile); - } else { - return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue); - } - } - - /** - * - * @param values the bucket values as longs - * @param geoValue the shape value - * @param precision the target precision to split the shape up into - * @return the number of buckets the geoValue is found in - */ - protected int setValuesByBruteForceScan(GeoShapeCellValues values, MultiGeoValues.GeoValue geoValue, - int precision, int minXTile, int minYTile, int maxXTile, int maxYTile) { - int idx = 0; - for (int i = minXTile; i <= maxXTile; i++) { - for (int j = minYTile; j <= maxYTile; j++) { - Rectangle rectangle = GeoTileUtils.toBoundingBox(i, j, precision); - if (geoValue.relate(rectangle) != GeoRelation.QUERY_DISJOINT) { - values.resizeCell(idx + 1); - values.add(idx++, GeoTileUtils.longEncodeTiles(precision, i, j)); - } - } - } - return idx; - } - - protected int setValuesByRasterization(int xTile, int yTile, int zTile, GeoShapeCellValues values, - int valuesIndex, int targetPrecision, MultiGeoValues.GeoValue geoValue) { - zTile++; - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int nextX = 2 * xTile + i; - int nextY = 2 * yTile + j; - Rectangle rectangle = GeoTileUtils.toBoundingBox(nextX, nextY, zTile); - GeoRelation relation = geoValue.relate(rectangle); - if (GeoRelation.QUERY_INSIDE == relation) { - if (zTile == targetPrecision) { - values.resizeCell(valuesIndex + 1); - values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); - } else { - values.resizeCell(valuesIndex + 1 << ( 2 * (targetPrecision - zTile)) + 1); - valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); - } - } else if (GeoRelation.QUERY_CROSSES == relation) { - if (zTile == targetPrecision) { - values.resizeCell(valuesIndex + 1); - values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); - } else { - valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, targetPrecision, geoValue); - } - } - } - } - return valuesIndex; - } - - private int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, - GeoShapeCellValues values, int valuesIndex, int targetPrecision) { - zTile++; - for (int i = 0; i < 2; i++) { - for (int j = 0; j < 2; j++) { - int nextX = 2 * xTile + i; - int nextY = 2 * yTile + j; - if (zTile == targetPrecision) { - values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); - } else { - valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); - } - } - } - return valuesIndex; - } + /** + * This sets the long-encoded value of the geo-point into the associated doc-values + * array. This is to be overridden by the {@link BoundedGeoTileGridTiler} and + * {@link BoundedGeoHashGridTiler} to check whether the point's tile intersects + * the appropriate bounds. + * + * @param values the doc-values array + * @param x the longitude of the point + * @param y the latitude of the point + * @param precision the zoom-level + * @param valuesIdx the index into the doc-values array at the time of advancement + * + * @return the next index into the array + */ + default int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + values[valuesIdx] = encode(x, y, precision); + return valuesIdx + 1; } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java index 153602493c3ba..0999732dc31d9 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorFactory.java @@ -80,7 +80,13 @@ protected Aggregator doCreateInternal(final ValuesSource.Geo valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, geoBoundingBox, GeoGridTiler.GeoHashGridTiler.INSTANCE); + final GeoGridTiler tiler; + if (geoBoundingBox.isUnbounded()) { + tiler = new GeoHashGridTiler(); + } else { + tiler = new BoundedGeoHashGridTiler(geoBoundingBox); + } + CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, tiler); return new GeoHashGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, searchContext, parent, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTiler.java new file mode 100644 index 0000000000000..7e60e7752d58c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridTiler.java @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +public class GeoHashGridTiler implements GeoGridTiler { + + @Override + public long encode(double x, double y, int precision) { + return Geohash.longEncode(x, y, precision); + } + + @Override + public int setValues(CellValues values, MultiGeoValues.GeoValue geoValue, int precision) { + if (precision == 1) { + values.resizeCell(1); + values.add(0, Geohash.longEncode(0, 0, 0)); + } + + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + assert bounds.minX() <= bounds.maxX(); + long numLonCells = (long) ((bounds.maxX() - bounds.minX()) / Geohash.lonWidthInDegrees(precision)); + long numLatCells = (long) ((bounds.maxY() - bounds.minY()) / Geohash.latHeightInDegrees(precision)); + long count = (numLonCells + 1) * (numLatCells + 1); + if (count == 1) { + return setValue(values, geoValue, bounds, precision); + } else if (count <= precision) { + return setValuesByBruteForceScan(values, geoValue, precision, bounds); + } else { + return setValuesByRasterization("", values, 0, precision, geoValue); + } + } + + /** + * Sets a singular doc-value for the {@link MultiGeoValues.GeoValue}. To be overriden by {@link BoundedGeoHashGridTiler} + * to account for {@link org.elasticsearch.common.geo.GeoBoundingBox} conditions + */ + protected int setValue(CellValues docValues, MultiGeoValues.GeoValue geoValue, MultiGeoValues.BoundingBox bounds, int precision) { + String hash = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + docValues.resizeCell(1); + docValues.add(0, Geohash.longEncode(hash)); + return 1; + } + + protected GeoRelation relateTile(MultiGeoValues.GeoValue geoValue, String hash) { + Rectangle rectangle = Geohash.toBoundingBox(hash); + return geoValue.relate(rectangle); + } + + protected int setValuesByBruteForceScan(CellValues values, MultiGeoValues.GeoValue geoValue, int precision, + MultiGeoValues.BoundingBox bounds) { + // TODO: This way to discover cells inside of a bounding box seems not to work as expected. I can + // see that eventually we will be visiting twice the same cell which should not happen. + int idx = 0; + String min = Geohash.stringEncode(bounds.minX(), bounds.minY(), precision); + String max = Geohash.stringEncode(bounds.maxX(), bounds.maxY(), precision); + String minNeighborBelow = Geohash.getNeighbor(min, precision, 0, -1); + double minY = Geohash.decodeLatitude((minNeighborBelow == null) ? min : minNeighborBelow); + double minX = Geohash.decodeLongitude(min); + double maxY = Geohash.decodeLatitude(max); + double maxX = Geohash.decodeLongitude(max); + for (double i = minX; i <= maxX; i += Geohash.lonWidthInDegrees(precision)) { + for (double j = minY; j <= maxY; j += Geohash.latHeightInDegrees(precision)) { + String hash = Geohash.stringEncode(i, j, precision); + GeoRelation relation = relateTile(geoValue, hash); + if (relation != GeoRelation.QUERY_DISJOINT) { + values.resizeCell(idx + 1); + values.add(idx++, encode(i, j, precision)); + } + } + } + return idx; + } + + protected int setValuesByRasterization(String hash, CellValues values, int valuesIndex, int targetPrecision, + MultiGeoValues.GeoValue geoValue) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + GeoRelation relation = relateTile(geoValue, hashes[i]); + if (relation == GeoRelation.QUERY_CROSSES) { + if (hashes[i].length() == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + valuesIndex = + setValuesByRasterization(hashes[i], values, valuesIndex, targetPrecision, geoValue); + } + } else if (relation == GeoRelation.QUERY_INSIDE) { + if (hashes[i].length() == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + values.resizeCell(valuesIndex + (int) Math.pow(32, targetPrecision - hash.length()) + 1); + valuesIndex = setValuesForFullyContainedTile(hashes[i],values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } + + protected int setValuesForFullyContainedTile(String hash, CellValues values, + int valuesIndex, int targetPrecision) { + String[] hashes = Geohash.getSubGeohashes(hash); + for (int i = 0; i < hashes.length; i++) { + if (hashes[i].length() == targetPrecision) { + values.add(valuesIndex++, Geohash.longEncode(hashes[i])); + } else { + valuesIndex = setValuesForFullyContainedTile(hashes[i], values, valuesIndex, targetPrecision); + } + } + return valuesIndex; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/UnboundedGeoPointCellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoPointCellValues.java similarity index 71% rename from server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/UnboundedGeoPointCellValues.java rename to server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoPointCellValues.java index 3ce9cac197158..57e7311eee38c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/UnboundedGeoPointCellValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoPointCellValues.java @@ -18,22 +18,19 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.index.fielddata.MultiGeoValues; /** - * Class representing {@link CellValues} that are unbounded by any - * {@link GeoBoundingBox}. + * Class representing geo_point {@link CellValues} */ -class UnboundedGeoPointCellValues extends CellValues { +class GeoPointCellValues extends CellValues { - protected UnboundedGeoPointCellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { + protected GeoPointCellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { super(geoValues, precision, tiler); } @Override int advanceValue(MultiGeoValues.GeoValue target, int valuesIdx) { - values[valuesIdx] = tiler.encode(target.lon(), target.lat(), precision); - return valuesIdx + 1; + return tiler.advancePointValue(values, target.lon(), target.lat(), precision, valuesIdx); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoShapeCellValues.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoShapeCellValues.java index 807b2808b9a45..22ed207a41338 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoShapeCellValues.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoShapeCellValues.java @@ -18,49 +18,18 @@ */ package org.elasticsearch.search.aggregations.bucket.geogrid; -import org.elasticsearch.index.fielddata.AbstractSortingNumericDocValues; import org.elasticsearch.index.fielddata.MultiGeoValues; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; - -import java.io.IOException; /** Sorted numeric doc values for geo shapes */ -class GeoShapeCellValues extends AbstractSortingNumericDocValues { - private MultiGeoValues geoValues; - private int precision; - private GeoGridTiler tiler; +class GeoShapeCellValues extends CellValues { protected GeoShapeCellValues(MultiGeoValues geoValues, int precision, GeoGridTiler tiler) { - this.geoValues = geoValues; - this.precision = precision; - this.tiler = tiler; - } - - protected void resizeCell(int newSize) { - resize(newSize); - } - - protected void add(int idx, long value) { - values[idx] = value; - } - - // for testing - protected long[] getValues() { - return values; + super(geoValues, precision, tiler); } @Override - public boolean advanceExact(int docId) throws IOException { - if (geoValues.advanceExact(docId)) { - ValuesSourceType vs = geoValues.valuesSourceType(); - MultiGeoValues.GeoValue target = geoValues.nextValue(); - // TODO(talevy): determine reasonable circuit-breaker here - resize(0); - tiler.setValues(this, target, precision); - sort(); - return true; - } else { - return false; - } + int advanceValue(MultiGeoValues.GeoValue target, int valuesIdx) { + // TODO(talevy): determine reasonable circuit-breaker here + return tiler.setValues(this, target, precision); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java index 54b29b01a8f20..680639e49a70d 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorFactory.java @@ -80,7 +80,13 @@ protected Aggregator doCreateInternal(final ValuesSource.Geo valuesSource, if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, searchContext, parent); } - CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, geoBoundingBox, GeoGridTiler.GeoTileGridTiler.INSTANCE); + final GeoGridTiler tiler; + if (geoBoundingBox.isUnbounded()) { + tiler = new GeoTileGridTiler(); + } else { + tiler = new BoundedGeoTileGridTiler(geoBoundingBox); + } + CellIdSource cellIdSource = new CellIdSource(valuesSource, precision, tiler); return new GeoTileGridAggregator(name, factories, cellIdSource, requiredSize, shardSize, searchContext, parent, pipelineAggregators, metaData); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTiler.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTiler.java new file mode 100644 index 0000000000000..869c14c24e252 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridTiler.java @@ -0,0 +1,166 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.search.aggregations.bucket.geogrid; + +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.fielddata.MultiGeoValues; + +public class GeoTileGridTiler implements GeoGridTiler { + + @Override + public long encode(double x, double y, int precision) { + return GeoTileUtils.longEncode(x, y, precision); + } + + public int advancePointValue(long[] values, double x, double y, int precision, int valuesIdx) { + values[valuesIdx] = encode(x, y, precision); + return valuesIdx + 1; + } + + /** + * Sets the values of the long[] underlying {@link CellValues}. + * + * If the shape resides between GeoTileUtils.NORMALIZED_LATITUDE_MASK and 90 or + * between GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK and -90 degree latitudes, then + * the shape is not accounted for since geo-tiles are only defined within those bounds. + * + * @param values the bucket values + * @param geoValue the input shape + * @param precision the tile zoom-level + * + * @return the number of tiles set by the shape + */ + @Override + public int setValues(CellValues values, MultiGeoValues.GeoValue geoValue, int precision) { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + assert bounds.minX() <= bounds.maxX(); + + if (precision == 0) { + values.resizeCell(1); + values.add(0, GeoTileUtils.longEncodeTiles(0, 0, 0)); + return 1; + } + + // geo tiles are not defined at the extreme latitudes due to them + // tiling the world as a square. + if ((bounds.top > GeoTileUtils.NORMALIZED_LATITUDE_MASK && bounds.bottom > GeoTileUtils.NORMALIZED_LATITUDE_MASK) + || (bounds.top < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK + && bounds.bottom < GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK)) { + return 0; + } + + final double tiles = 1 << precision; + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + int count = (maxXTile - minXTile + 1) * (maxYTile - minYTile + 1); + if (count == 1) { + return setValue(values, geoValue, minXTile, minYTile, precision); + } else if (count <= precision) { + return setValuesByBruteForceScan(values, geoValue, precision, minXTile, minYTile, maxXTile, maxYTile); + } else { + return setValuesByRasterization(0, 0, 0, values, 0, precision, geoValue); + } + } + + protected GeoRelation relateTile(MultiGeoValues.GeoValue geoValue, int xTile, int yTile, int precision) { + Rectangle rectangle = GeoTileUtils.toBoundingBox(xTile, yTile, precision); + return geoValue.relate(rectangle); + } + + /** + * Sets a singular doc-value for the {@link MultiGeoValues.GeoValue}. To be overriden by {@link BoundedGeoTileGridTiler} + * to account for {@link org.elasticsearch.common.geo.GeoBoundingBox} conditions + */ + protected int setValue(CellValues docValues, MultiGeoValues.GeoValue geoValue, int xTile, int yTile, int precision) { + docValues.resizeCell(1); + docValues.add(0, GeoTileUtils.longEncodeTiles(precision, xTile, yTile)); + return 1; + } + + /** + * + * @param values the bucket values as longs + * @param geoValue the shape value + * @param precision the target precision to split the shape up into + * @return the number of buckets the geoValue is found in + */ + protected int setValuesByBruteForceScan(CellValues values, MultiGeoValues.GeoValue geoValue, + int precision, int minXTile, int minYTile, int maxXTile, int maxYTile) { + int idx = 0; + for (int i = minXTile; i <= maxXTile; i++) { + for (int j = minYTile; j <= maxYTile; j++) { + GeoRelation relation = relateTile(geoValue, i, j, precision); + if (relation != GeoRelation.QUERY_DISJOINT) { + values.resizeCell(idx + 1); + values.add(idx++, GeoTileUtils.longEncodeTiles(precision, i, j)); + } + } + } + return idx; + } + + protected int setValuesByRasterization(int xTile, int yTile, int zTile, CellValues values, int valuesIndex, + int targetPrecision, MultiGeoValues.GeoValue geoValue) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + GeoRelation relation = relateTile(geoValue, nextX, nextY, zTile); + if (GeoRelation.QUERY_INSIDE == relation) { + if (zTile == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + values.resizeCell(valuesIndex + 1 << ( 2 * (targetPrecision - zTile)) + 1); + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } else if (GeoRelation.QUERY_CROSSES == relation) { + if (zTile == targetPrecision) { + values.resizeCell(valuesIndex + 1); + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + valuesIndex = setValuesByRasterization(nextX, nextY, zTile, values, valuesIndex, targetPrecision, geoValue); + } + } + } + } + return valuesIndex; + } + + protected int setValuesForFullyContainedTile(int xTile, int yTile, int zTile, CellValues values, int valuesIndex, + int targetPrecision) { + zTile++; + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 2; j++) { + int nextX = 2 * xTile + i; + int nextY = 2 * yTile + j; + if (zTile == targetPrecision) { + values.add(valuesIndex++, GeoTileUtils.longEncodeTiles(zTile, nextX, nextY)); + } else { + valuesIndex = setValuesForFullyContainedTile(nextX, nextY, zTile, values, valuesIndex, targetPrecision); + } + } + } + return valuesIndex; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java index c97bb88135109..d872a8c7ce640 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileUtils.java @@ -44,23 +44,10 @@ */ public final class GeoTileUtils { - /** - * The geo-tile map is clipped at 85.05112878 to 90 and -85.05112878 to -90 - */ - public static final double LATITUDE_MASK = 85.0511287798066; - - /** - * Since shapes are encoded, their boundaries are to be compared to against the encoded/decoded values of LATITUDE_MASK - */ - static final double NORMALIZED_LATITUDE_MASK = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(LATITUDE_MASK)); - static final double NORMALIZED_NEGATIVE_LATITUDE_MASK = - GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(-LATITUDE_MASK)); + private GeoTileUtils() {} private static final double PI_DIV_2 = Math.PI / 2; - - private GeoTileUtils() {} - /** * Largest number of tiles (precision) to use. * This value cannot be more than (64-5)/2 = 29, because 5 bits are used for zoom level itself (0-31) @@ -71,6 +58,18 @@ private GeoTileUtils() {} */ public static final int MAX_ZOOM = 29; + /** + * The geo-tile map is clipped at 85.05112878 to 90 and -85.05112878 to -90 + */ + public static final double LATITUDE_MASK = 85.0511287798066; + + /** + * Since shapes are encoded, their boundaries are to be compared to against the encoded/decoded values of LATITUDE_MASK + */ + static final double NORMALIZED_LATITUDE_MASK = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(LATITUDE_MASK)); + static final double NORMALIZED_NEGATIVE_LATITUDE_MASK = + GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(-LATITUDE_MASK)); + /** * Bit position of the zoom value within hash - zoom is stored in the most significant 6 bits of a long number. */ @@ -81,6 +80,7 @@ private GeoTileUtils() {} */ private static final long X_Y_VALUE_MASK = (1L << MAX_ZOOM) - 1; + /** * Parse an integer precision (zoom level). The {@link ValueType#INT} allows it to be a number or a string. * @@ -246,6 +246,11 @@ static GeoPoint keyToGeoPoint(String hashAsString) { return zxyToGeoPoint(hashAsInts[0], hashAsInts[1], hashAsInts[2]); } + static Rectangle toBoundingBox(long hash) { + int[] hashAsInts = parseHash(hash); + return toBoundingBox(hashAsInts[1], hashAsInts[2], hashAsInts[0]); + } + static Rectangle toBoundingBox(int xTile, int yTile, int precision) { final double tiles = validateZXY(precision, xTile, yTile); final double minN = Math.PI - (2.0 * Math.PI * (yTile + 1)) / tiles; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java index 15585c2f4a3a8..22249d8aa6a94 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoBounds.java @@ -175,7 +175,8 @@ public XContentBuilder doXContentBody(XContentBuilder builder, Params params) th return builder; } - private GeoBoundingBox resolveGeoBoundingBox() { + // used for testing + GeoBoundingBox resolveGeoBoundingBox() { if (Double.isInfinite(top)) { return null; } else if (Double.isInfinite(posLeft)) { diff --git a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java index 3de9d70d459ab..f7561508f831d 100644 --- a/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/TriangleTreeTests.java @@ -257,9 +257,7 @@ public void testPacManPoints() throws Exception { assertRelation(GeoRelation.QUERY_CROSSES, reader, getExtentFromBox(xMin, yMin, xMax, yMax)); } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") public void testRandomMultiLineIntersections() throws IOException { - double extentSize = randomDoubleBetween(0.01, 10, true); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); MultiLine geometry = randomMultiLine(false); geometry = (MultiLine) indexer.prepareForIndexing(geometry); @@ -267,14 +265,7 @@ public void testRandomMultiLineIntersections() throws IOException { Extent readerExtent = reader.getExtent(); for (Line line : geometry) { - // extent that intersects edges - assertRelation(GeoRelation.QUERY_CROSSES, reader, bufferedExtentFromGeoPoint(line.getX(0), line.getY(0), extentSize)); - - // TODO(talevy): resolve definition. when line is on a specific edge it is not considered crossing due to latest changes - // extent that fully encloses a line in the MultiLine Extent lineExtent = triangleTreeReader(line, GeoShapeCoordinateEncoder.INSTANCE).getExtent(); - assertRelation(GeoRelation.QUERY_CROSSES, reader, lineExtent); - if (lineExtent.minX() != Integer.MIN_VALUE && lineExtent.maxX() != Integer.MAX_VALUE && lineExtent.minY() != Integer.MIN_VALUE && lineExtent.maxY() != Integer.MAX_VALUE) { assertRelation(GeoRelation.QUERY_CROSSES, reader, Extent.fromPoints(lineExtent.minX() - 1, lineExtent.minY() - 1, @@ -292,9 +283,8 @@ public void testRandomMultiLineIntersections() throws IOException { } - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/37206") - public void testRandomGeometryIntersection() throws IOException { - int testPointCount = randomIntBetween(100, 200); + public void testRandomPolygonIntersection() throws IOException { + int testPointCount = randomIntBetween(50, 100); Point[] testPoints = new Point[testPointCount]; double extentSize = randomDoubleBetween(1, 10, true); boolean[] intersects = new boolean[testPointCount]; @@ -302,7 +292,7 @@ public void testRandomGeometryIntersection() throws IOException { testPoints[i] = randomPoint(false); } - Geometry geometry = randomGeometryTreeGeometry(); + Geometry geometry = randomMultiPolygon(false); GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); Geometry preparedGeometry = indexer.prepareForIndexing(geometry); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java index 3840b3423a94e..42417db2484db 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregatorTestCase.java @@ -30,14 +30,17 @@ import org.apache.lucene.store.Directory; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.geo.CentroidCalculator; +import org.elasticsearch.common.geo.GeoRelation; +import org.elasticsearch.common.geo.GeoShapeCoordinateEncoder; import org.elasticsearch.common.geo.GeoTestUtils; +import org.elasticsearch.common.geo.TriangleTreeReader; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.Point; -import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoBoundingBoxTests; -import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.index.mapper.GeoPointFieldMapper; import org.elasticsearch.index.mapper.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -54,14 +57,13 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Function; +import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; import static org.hamcrest.Matchers.equalTo; public abstract class GeoGridAggregatorTestCase extends AggregatorTestCase { private static final String FIELD_NAME = "location"; - protected static final double GEOHASH_TOLERANCE = 1E-5D; /** * Generate a random precision according to the rules of the given aggregation. @@ -73,6 +75,16 @@ public abstract class GeoGridAggregatorTestCase */ protected abstract String hashAsString(double lng, double lat, int precision); + /** + * Return a point within the bounds of the tile grid + */ + protected abstract Point randomPoint(); + + /** + * Return the bounding tile as a {@link Rectangle} for a given point + */ + protected abstract Rectangle getTile(double lng, double lat, int precision); + /** * Create a new named {@link GeoGridAggregationBuilder}-derived builder */ @@ -175,57 +187,55 @@ public void testGeoPointWithSeveralDocs() throws IOException { }, new GeoPointFieldMapper.GeoPointFieldType()); } - public void testBounds() throws IOException { - final int numDocs = randomIntBetween(64, 256); + public void testGeoPointBounds() throws IOException { + final int precision = randomPrecision(); + final int numDocs = randomIntBetween(100, 200); + int numDocsWithin = 0; final GeoGridAggregationBuilder builder = createBuilder("_name"); expectThrows(IllegalArgumentException.class, () -> builder.precision(-1)); expectThrows(IllegalArgumentException.class, () -> builder.precision(30)); - // only consider bounding boxes that are at least GEOHASH_TOLERANCE wide and have quantized coordinates - GeoBoundingBox bbox = randomValueOtherThanMany( - (b) -> Math.abs(GeoUtils.normalizeLon(b.right()) - GeoUtils.normalizeLon(b.left())) < GEOHASH_TOLERANCE, - GeoBoundingBoxTests::randomBBox); - Function encodeDecodeLat = (lat) -> GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(lat)); - Function encodeDecodeLon = (lon) -> GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(lon)); - bbox.topLeft().reset(encodeDecodeLat.apply(bbox.top()), encodeDecodeLon.apply(bbox.left())); - bbox.bottomRight().reset(encodeDecodeLat.apply(bbox.bottom()), encodeDecodeLon.apply(bbox.right())); + GeoBoundingBox bbox = GeoBoundingBoxTests.randomBBox(); + final double boundsTop = bbox.top(); + final double boundsBottom = bbox.bottom(); + final double boundsWestLeft; + final double boundsWestRight; + final double boundsEastLeft; + final double boundsEastRight; + final boolean crossesDateline; + if (bbox.right() < bbox.left()) { + boundsWestLeft = -180; + boundsWestRight = bbox.right(); + boundsEastLeft = bbox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = bbox.left(); + boundsEastRight = bbox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } - int in = 0, out = 0; List docs = new ArrayList<>(); - while (in + out < numDocs) { - if (bbox.left() > bbox.right()) { - if (randomBoolean()) { - double lonWithin = randomBoolean() ? - randomDoubleBetween(bbox.left(), 180.0, true) - : randomDoubleBetween(-180.0, bbox.right(), true); - double latWithin = randomDoubleBetween(bbox.bottom(), bbox.top(), true); - in++; - docs.add(new LatLonDocValuesField(FIELD_NAME, latWithin, lonWithin)); - } else { - double lonOutside = randomDoubleBetween(bbox.left(), bbox.right(), true); - double latOutside = randomDoubleBetween(bbox.top(), -90, false); - out++; - docs.add(new LatLonDocValuesField(FIELD_NAME, latOutside, lonOutside)); - } - } else { - if (randomBoolean()) { - double lonWithin = randomDoubleBetween(bbox.left(), bbox.right(), true); - double latWithin = randomDoubleBetween(bbox.bottom(), bbox.top(), true); - in++; - docs.add(new LatLonDocValuesField(FIELD_NAME, latWithin, lonWithin)); - } else { - double lonOutside = GeoUtils.normalizeLon(randomDoubleBetween(bbox.right(), 180.001, false)); - double latOutside = GeoUtils.normalizeLat(randomDoubleBetween(bbox.top(), 90.001, false)); - out++; - docs.add(new LatLonDocValuesField(FIELD_NAME, latOutside, lonOutside)); - } + for (int i = 0; i < numDocs; i++) { + Point p; + p = randomPoint(); + double x = GeoTestUtils.encodeDecodeLon(p.getX()); + double y = GeoTestUtils.encodeDecodeLat(p.getY()); + Rectangle pointTile = getTile(x, y, precision); + + boolean intersectsBounds = boundsTop >= pointTile.getMinY() && boundsBottom <= pointTile.getMaxY() + && (boundsEastLeft <= pointTile.getMaxX() && boundsEastRight >= pointTile.getMinX() + || (crossesDateline && boundsWestLeft <= pointTile.getMaxX() && boundsWestRight >= pointTile.getMinX())); + if (intersectsBounds) { + numDocsWithin += 1; } - + docs.add(new LatLonDocValuesField(FIELD_NAME, p.getLat(), p.getLon())); } - final long numDocsInBucket = in; - final int precision = randomPrecision(); + final long numDocsInBucket = numDocsWithin; testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, bbox, iw -> { for (LatLonDocValuesField docField : docs) { @@ -242,6 +252,82 @@ public void testBounds() throws IOException { }, new GeoPointFieldMapper.GeoPointFieldType()); } + public void testGeoShapeBounds() throws IOException { + final int precision = randomPrecision(); + final int numDocs = randomIntBetween(100, 200); + int numDocsWithin = 0; + final GeoGridAggregationBuilder builder = createBuilder("_name"); + + expectThrows(IllegalArgumentException.class, () -> builder.precision(-1)); + expectThrows(IllegalArgumentException.class, () -> builder.precision(30)); + + GeoBoundingBox bbox = GeoBoundingBoxTests.randomBBox(); + final double boundsTop = bbox.top(); + final double boundsBottom = bbox.bottom(); + final double boundsWestLeft; + final double boundsWestRight; + final double boundsEastLeft; + final double boundsEastRight; + final boolean crossesDateline; + if (bbox.right() < bbox.left()) { + boundsWestLeft = -180; + boundsWestRight = bbox.right(); + boundsEastLeft = bbox.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { // only set east bounds + boundsEastLeft = bbox.left(); + boundsEastRight = bbox.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + + List docs = new ArrayList<>(); + List points = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + Point p; + p = randomPoint(); + double x = GeoTestUtils.encodeDecodeLon(p.getX()); + double y = GeoTestUtils.encodeDecodeLat(p.getY()); + Rectangle pointTile = getTile(x, y, precision); + + + TriangleTreeReader reader = triangleTreeReader(p, GeoShapeCoordinateEncoder.INSTANCE); + GeoRelation tileRelation = reader.relateTile(GeoShapeCoordinateEncoder.INSTANCE.encodeX(pointTile.getMinX()), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(pointTile.getMinY()), + GeoShapeCoordinateEncoder.INSTANCE.encodeX(pointTile.getMaxX()), + GeoShapeCoordinateEncoder.INSTANCE.encodeY(pointTile.getMaxY())); + boolean intersectsBounds = boundsTop >= pointTile.getMinY() && boundsBottom <= pointTile.getMaxY() + && (boundsEastLeft <= pointTile.getMaxX() && boundsEastRight >= pointTile.getMinX() + || (crossesDateline && boundsWestLeft <= pointTile.getMaxX() && boundsWestRight >= pointTile.getMinX())); + if (tileRelation != GeoRelation.QUERY_DISJOINT && intersectsBounds) { + numDocsWithin += 1; + } + + + points.add(p); + docs.add(new BinaryGeoShapeDocValuesField(FIELD_NAME, + GeoTestUtils.toDecodedTriangles(p), new CentroidCalculator(p))); + } + + final long numDocsInBucket = numDocsWithin; + + testCase(new MatchAllDocsQuery(), FIELD_NAME, precision, bbox, iw -> { + for (BinaryGeoShapeDocValuesField docField : docs) { + iw.addDocument(Collections.singletonList(docField)); + } + }, + geoGrid -> { + assertThat(AggregationInspectionHelper.hasValue(geoGrid), equalTo(numDocsInBucket > 0)); + long docCount = 0; + for (int i = 0; i < geoGrid.getBuckets().size(); i++) { + docCount += geoGrid.getBuckets().get(i).getDocCount(); + } + assertThat(docCount, equalTo(numDocsInBucket)); + }, new GeoShapeFieldMapper.GeoShapeFieldType()); + } + public void testGeoShapeWithSeveralDocs() throws IOException { int precision = randomIntBetween(1, 4); int numShapes = randomIntBetween(8, 128); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java index 8d3f8b04aa7d3..9c64dccfafff9 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridTilerTests.java @@ -43,13 +43,15 @@ import static org.elasticsearch.common.geo.GeoTestUtils.encodeDecodeLat; import static org.elasticsearch.common.geo.GeoTestUtils.encodeDecodeLon; import static org.elasticsearch.common.geo.GeoTestUtils.triangleTreeReader; +import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.LATITUDE_MASK; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_LATITUDE_MASK; import static org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils.NORMALIZED_NEGATIVE_LATITUDE_MASK; + import static org.hamcrest.Matchers.equalTo; public class GeoGridTilerTests extends ESTestCase { - private static final GeoGridTiler.GeoTileGridTiler GEOTILE = GeoGridTiler.GeoTileGridTiler.INSTANCE; - private static final GeoGridTiler.GeoHashGridTiler GEOHASH = GeoGridTiler.GeoHashGridTiler.INSTANCE; + private static final GeoTileGridTiler GEOTILE = new GeoTileGridTiler(); + private static final GeoHashGridTiler GEOHASH = new GeoHashGridTiler(); public void testGeoTile() throws Exception { double x = randomDouble(); @@ -82,6 +84,65 @@ public void testGeoTile() throws Exception { } } + public void testAdvancePointValue() { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(1, 6); + int size = randomIntBetween(1, 10); + long[] values = new long[size]; + int idx = randomIntBetween(0, size - 1); + Point point = GeometryTestUtils.randomPoint(false); + for (GeoGridTiler tiler : List.of(GEOTILE, GEOHASH)) { + int newIdx = tiler.advancePointValue(values, point.getX(), point.getY(), precision, idx); + assertThat(newIdx, equalTo(idx + 1)); + assertThat(values[idx], equalTo(tiler.encode(point.getX(), point.getY(), precision))); + } + } + } + + public void testBoundedGeotileAdvancePointValue() { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(1, 6); + int size = randomIntBetween(1, 10); + long[] values = new long[size]; + int idx = randomIntBetween(0, size - 1); + Point point = GeometryTestUtils.randomPoint(false); + GeoBoundingBox geoBoundingBox = GeoBoundingBoxTests.randomBBox(); + + BoundedGeoTileGridTiler tiler = new BoundedGeoTileGridTiler(geoBoundingBox); + int newIdx = tiler.advancePointValue(values, point.getX(), point.getY(), precision, idx); + if (newIdx == idx + 1) { + assertTrue(tiler.cellIntersectsGeoBoundingBox(GeoTileUtils.toBoundingBox(values[idx]))); + assertThat(values[idx], equalTo(tiler.encode(point.getX(), point.getY(), precision))); + assertThat(newIdx, equalTo(idx + 1)); + } else { + assertThat(newIdx, equalTo(idx)); + assertThat(values[idx], equalTo(0L)); + } + } + } + + public void testBoundedGeohashAdvancePointValue() { + for (int i = 0; i < 100; i++) { + int precision = randomIntBetween(1, 6); + int size = randomIntBetween(1, 10); + long[] values = new long[size]; + int idx = randomIntBetween(0, size - 1); + Point point = GeometryTestUtils.randomPoint(false); + GeoBoundingBox geoBoundingBox = GeoBoundingBoxTests.randomBBox(); + + BoundedGeoHashGridTiler tiler = new BoundedGeoHashGridTiler(geoBoundingBox); + int newIdx = tiler.advancePointValue(values, point.getX(), point.getY(), precision, idx); + if (newIdx == idx + 1) { + assertTrue(tiler.cellIntersectsGeoBoundingBox(Geohash.toBoundingBox(Geohash.stringEncode(values[idx])))); + assertThat(values[idx], equalTo(tiler.encode(point.getX(), point.getY(), precision))); + assertThat(newIdx, equalTo(idx + 1)); + } else { + assertThat(newIdx, equalTo(idx)); + assertThat(values[idx], equalTo(0L)); + } + } + } + public void testGeoTileSetValuesBruteAndRecursiveMultiline() throws Exception { MultiLine geometry = GeometryTestUtils.randomMultiLine(false); checkGeoTileSetValuesBruteAndRecursive(geometry); @@ -100,61 +161,30 @@ public void testGeoTileSetValuesBruteAndRecursivePoints() throws Exception { checkGeoHashSetValuesBruteAndRecursive(geometry); } - private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Exception { - int precision = randomIntBetween(1, 5); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - geometry = indexer.prepareForIndexing(geometry); - TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); - MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); - GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOTILE); - int recursiveCount; - { - recursiveCount = GEOTILE.setValuesByRasterization(0, 0, 0, recursiveValues, 0, precision, value); - } - GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOTILE); - int bruteForceCount; - { - final double tiles = 1 << precision; - MultiGeoValues.BoundingBox bounds = value.boundingBox(); - int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); - int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); - int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); - int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); - bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision, minXTile, minYTile, maxXTile, maxYTile); - } - assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); - long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); - long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); - Arrays.sort(recursive); - Arrays.sort(bruteForce); - assertArrayEquals(geometry.toString(), recursive, bruteForce); - } + // tests that bounding boxes of shapes crossing the dateline are correctly wrapped + public void testGeoTileSetValuesBoundingBoxes_BoundedGeoShapeCellValues() throws Exception { + for (int i = 0; i < 1; i++) { + int precision = randomIntBetween(0, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + Geometry geometry = indexer.prepareForIndexing(randomValueOtherThanMany(g -> { + try { + indexer.prepareForIndexing(g); + return false; + } catch (Exception e) { + return true; + } + }, () -> boxToGeo(GeoBoundingBoxTests.randomBBox()))); - private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Exception { - int precision = randomIntBetween(1, 3); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - geometry = indexer.prepareForIndexing(geometry); - TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); - MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); - GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOHASH); - int recursiveCount; - { - recursiveCount = GEOHASH.setValuesByRasterization("", recursiveValues, 0, precision, value); - } - GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOHASH); - int bruteForceCount; - { - MultiGeoValues.BoundingBox bounds = value.boundingBox(); - bruteForceCount = GEOHASH.setValuesByBruteForceScan(bruteForceValues, value, precision, bounds); - } + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + GeoBoundingBox geoBoundingBox = GeoBoundingBoxTests.randomBBox(); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues cellValues = new GeoShapeCellValues(null, precision, GEOTILE); - assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + int numTiles = new BoundedGeoTileGridTiler(geoBoundingBox).setValues(cellValues, value, precision); + int expected = numTiles(value, precision, geoBoundingBox); - long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); - long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); - Arrays.sort(recursive); - Arrays.sort(bruteForce); - assertArrayEquals(geometry.toString(), recursive, bruteForce); + assertThat(numTiles, equalTo(expected)); + } } // test random rectangles that can cross the date-line and verify that there are an expected @@ -174,7 +204,7 @@ public void testGeoTileSetValuesBoundingBoxes_UnboundedGeoShapeCellValues() thro TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); - GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); + CellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); int numTiles = GEOTILE.setValues(unboundedCellValues, value, precision); int expected = numTiles(value, precision); assertThat(numTiles, equalTo(expected)); @@ -202,6 +232,9 @@ public void testTilerMatchPoint() throws Exception { }; for (Point point : pointCorners) { + if (point.getX() == GeoUtils.MAX_LON || point.getY() == -LATITUDE_MASK) { + continue; + } TriangleTreeReader reader = triangleTreeReader(point, GeoShapeCoordinateEncoder.INSTANCE); MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); GeoShapeCellValues unboundedCellValues = new GeoShapeCellValues(null, precision, GEOTILE); @@ -216,7 +249,7 @@ public void testTilerMatchPoint() throws Exception { public void testGeoHash() throws Exception { double x = randomDouble(); double y = randomDouble(); - int precision = randomIntBetween(0, Geohash.PRECISION); + int precision = randomIntBetween(0, 6); assertThat(GEOHASH.encode(x, y, precision), equalTo(Geohash.longEncode(x, y, precision))); Rectangle tile = Geohash.toBoundingBox(Geohash.stringEncode(x, y, 5)); @@ -228,23 +261,175 @@ public void testGeoHash() throws Exception { // test shape within tile bounds { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); int count = GEOHASH.setValues(values, value, 5); assertThat(count, equalTo(1)); } { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); int count = GEOHASH.setValues(values, value, 6); assertThat(count, equalTo(32)); } { - GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOTILE); + GeoShapeCellValues values = new GeoShapeCellValues(null, precision, GEOHASH); int count = GEOHASH.setValues(values, value, 7); assertThat(count, equalTo(1024)); } } - private Geometry boxToGeo(GeoBoundingBox geoBox) { + private boolean tileIntersectsBounds(int x, int y, int precision, GeoBoundingBox bounds) { + if (bounds == null) { + return true; + } + final double boundsWestLeft; + final double boundsWestRight; + final double boundsEastLeft; + final double boundsEastRight; + final boolean crossesDateline; + if (bounds.right() < bounds.left()) { + boundsWestLeft = -180; + boundsWestRight = bounds.right(); + boundsEastLeft = bounds.left(); + boundsEastRight = 180; + crossesDateline = true; + } else { + boundsEastLeft = bounds.left(); + boundsEastRight = bounds.right(); + boundsWestLeft = 0; + boundsWestRight = 0; + crossesDateline = false; + } + + Rectangle tile = GeoTileUtils.toBoundingBox(x, y, precision); + + return (bounds.top() >= tile.getMinY() && bounds.bottom() <= tile.getMaxY() + && (boundsEastLeft <= tile.getMaxX() && boundsEastRight >= tile.getMinX() + || (crossesDateline && boundsWestLeft <= tile.getMaxX() && boundsWestRight >= tile.getMinX()))); + } + + private int numTiles(MultiGeoValues.GeoValue geoValue, int precision, GeoBoundingBox geoBox) throws Exception { + MultiGeoValues.BoundingBox bounds = geoValue.boundingBox(); + int count = 0; + + if (precision == 0) { + return 1; + } else if ((bounds.top > LATITUDE_MASK && bounds.bottom > LATITUDE_MASK) + || (bounds.top < -LATITUDE_MASK && bounds.bottom < -LATITUDE_MASK)) { + return 0; + } + final double tiles = 1 << precision; + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + if ((bounds.posLeft >= 0 && bounds.posRight >= 0) && (bounds.negLeft < 0 && bounds.negRight < 0)) { + // box one + int minXTileNeg = GeoTileUtils.getXTile(bounds.negLeft, (long) tiles); + int maxXTileNeg = GeoTileUtils.getXTile(bounds.negRight, (long) tiles); + + for (int x = minXTileNeg; x <= maxXTileNeg; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + + // box two + int minXTilePos = GeoTileUtils.getXTile(bounds.posLeft, (long) tiles); + if (minXTilePos > maxXTileNeg + 1) { + minXTilePos -= 1; + } + + int maxXTilePos = GeoTileUtils.getXTile(bounds.posRight, (long) tiles); + + for (int x = minXTilePos; x <= maxXTilePos; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } else { + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + + if (minXTile == maxXTile && minYTile == maxYTile) { + return tileIntersectsBounds(minXTile, minYTile, precision, geoBox) ? 1 : 0; + } + + for (int x = minXTile; x <= maxXTile; x++) { + for (int y = minYTile; y <= maxYTile; y++) { + Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); + if (tileIntersectsBounds(x, y, precision, geoBox) && geoValue.relate(r) != GeoRelation.QUERY_DISJOINT) { + count += 1; + } + } + } + return count; + } + } + + private void checkGeoTileSetValuesBruteAndRecursive(Geometry geometry) throws Exception { + int precision = randomIntBetween(1, 4); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOTILE); + int recursiveCount; + { + recursiveCount = GEOTILE.setValuesByRasterization(0, 0, 0, recursiveValues, 0, precision, value); + } + GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOTILE); + int bruteForceCount; + { + final double tiles = 1 << precision; + MultiGeoValues.BoundingBox bounds = value.boundingBox(); + int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); + int minYTile = GeoTileUtils.getYTile(bounds.maxY(), (long) tiles); + int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + int maxYTile = GeoTileUtils.getYTile(bounds.minY(), (long) tiles); + bruteForceCount = GEOTILE.setValuesByBruteForceScan(bruteForceValues, value, precision, minXTile, minYTile, maxXTile, maxYTile); + } + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); + long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); + Arrays.sort(recursive); + Arrays.sort(bruteForce); + assertArrayEquals(geometry.toString(), recursive, bruteForce); + } + + private void checkGeoHashSetValuesBruteAndRecursive(Geometry geometry) throws Exception { + int precision = randomIntBetween(1, 3); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + geometry = indexer.prepareForIndexing(geometry); + TriangleTreeReader reader = triangleTreeReader(geometry, GeoShapeCoordinateEncoder.INSTANCE); + MultiGeoValues.GeoShapeValue value = new MultiGeoValues.GeoShapeValue(reader); + GeoShapeCellValues recursiveValues = new GeoShapeCellValues(null, precision, GEOHASH); + int recursiveCount; + { + recursiveCount = GEOHASH.setValuesByRasterization("", recursiveValues, 0, precision, value); + } + GeoShapeCellValues bruteForceValues = new GeoShapeCellValues(null, precision, GEOHASH); + int bruteForceCount; + { + MultiGeoValues.BoundingBox bounds = value.boundingBox(); + bruteForceCount = GEOHASH.setValuesByBruteForceScan(bruteForceValues, value, precision, bounds); + } + + assertThat(geometry.toString(), recursiveCount, equalTo(bruteForceCount)); + + long[] recursive = Arrays.copyOf(recursiveValues.getValues(), recursiveCount); + long[] bruteForce = Arrays.copyOf(bruteForceValues.getValues(), bruteForceCount); + Arrays.sort(recursive); + Arrays.sort(bruteForce); + assertArrayEquals(geometry.toString(), recursive, bruteForce); + } + + + static Geometry boxToGeo(GeoBoundingBox geoBox) { // turn into polygon if (geoBox.right() < geoBox.left() && geoBox.right() != -180) { return new MultiPolygon(List.of( @@ -313,6 +498,11 @@ private int numTiles(MultiGeoValues.GeoValue geoValue, int precision) { } else { int minXTile = GeoTileUtils.getXTile(bounds.minX(), (long) tiles); int maxXTile = GeoTileUtils.getXTile(bounds.maxX(), (long) tiles); + + if (minXTile == maxXTile && minYTile == maxYTile) { + return 1; + } + for (int x = minXTile; x <= maxXTile; x++) { for (int y = minYTile; y <= maxYTile; y++) { Rectangle r = GeoTileUtils.toBoundingBox(x, y, precision); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java index 7fa517807f619..2aab993c1d8c0 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoHashGridAggregatorTests.java @@ -19,6 +19,11 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.geometry.utils.Geohash; + import static org.elasticsearch.geometry.utils.Geohash.stringEncode; public class GeoHashGridAggregatorTests extends GeoGridAggregatorTestCase { @@ -33,6 +38,16 @@ protected String hashAsString(double lng, double lat, int precision) { return stringEncode(lng, lat, precision); } + @Override + protected Point randomPoint() { + return GeometryTestUtils.randomPoint(false); + } + + @Override + protected Rectangle getTile(double lng, double lat, int precision) { + return Geohash.toBoundingBox(Geohash.stringEncode(lng, lat, precision)); + } + @Override protected GeoGridAggregationBuilder createBuilder(String name) { return new GeoHashGridAggregationBuilder(name); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java index 85b2306403230..fd4c52a0f7d8d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoTileGridAggregatorTests.java @@ -19,6 +19,10 @@ package org.elasticsearch.search.aggregations.bucket.geogrid; +import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; + public class GeoTileGridAggregatorTests extends GeoGridAggregatorTestCase { @Override @@ -31,6 +35,17 @@ protected String hashAsString(double lng, double lat, int precision) { return GeoTileUtils.stringEncode(GeoTileUtils.longEncode(lng, lat, precision)); } + @Override + protected Point randomPoint() { + return new Point(randomDoubleBetween(GeoUtils.MIN_LON, GeoUtils.MAX_LON, true), + randomDoubleBetween(-GeoTileUtils.LATITUDE_MASK, GeoTileUtils.LATITUDE_MASK, false)); + } + + @Override + protected Rectangle getTile(double lng, double lat, int precision) { + return GeoTileUtils.toBoundingBox(GeoTileUtils.longEncode(lng, lat, precision)); + } + @Override protected GeoGridAggregationBuilder createBuilder(String name) { return new GeoTileGridAggregationBuilder(name); From b773b46a613fc1850c1c07977875add30f81c60d Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Thu, 5 Mar 2020 08:01:18 -0800 Subject: [PATCH 62/62] add explicit tests for geo_shape with doc values on old indices (#52861) --- .../mapper/GeoShapeFieldMapperTests.java | 26 +++++++ .../aggregations/bucket/GeoHashGridIT.java | 78 +++++++++++++++---- .../metrics/AbstractGeoTestCase.java | 21 ++++- .../aggregations/metrics/GeoBoundsIT.java | 26 +++++++ .../aggregations/metrics/GeoCentroidIT.java | 28 +++++++ 5 files changed, 161 insertions(+), 18 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index a2c20762097b2..751b0deb57185 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.index.mapper; +import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; @@ -28,6 +29,7 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.util.Collection; @@ -40,6 +42,11 @@ public class GeoShapeFieldMapperTests extends ESSingleNodeTestCase { + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + @Override protected Collection> getPlugins() { return pluginList(InternalSettingsPlugin.class); @@ -65,6 +72,25 @@ public void testDefaultConfiguration() throws IOException { assertTrue(geoShapeFieldMapper.fieldType().hasDocValues()); } + public void testDefaultDocValueConfigurationOnPre8() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("location") + .field("type", "geo_shape") + .endObject().endObject() + .endObject().endObject()); + + Version oldVersion = VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0); + DocumentMapper defaultMapper = createIndex("test", settings(oldVersion).build()).mapperService().documentMapperParser() + .parse("type1", new CompressedXContent(mapping)); + Mapper fieldMapper = defaultMapper.mappers().getMapper("location"); + assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class)); + + GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper; + assertFalse(geoShapeFieldMapper.docValues().explicit()); + assertFalse(geoShapeFieldMapper.docValues().value()); + assertFalse(geoShapeFieldMapper.fieldType().hasDocValues()); + } + /** * Test that orientation parameter correctly parses */ diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java index 61f6c60424671..6d11f93616f41 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/GeoHashGridIT.java @@ -23,12 +23,16 @@ import com.carrotsearch.hppc.cursors.ObjectIntCursor; import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.GeographyValidator; +import org.elasticsearch.geometry.utils.WellKnownText; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.filter.Filter; @@ -43,6 +47,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.geometry.utils.Geohash.PRECISION; import static org.elasticsearch.geometry.utils.Geohash.stringEncode; @@ -52,44 +57,46 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; @ESIntegTestCase.SuiteScopeTestCase public class GeoHashGridIT extends ESIntegTestCase { + private static WellKnownText WKT = new WellKnownText(false, new GeographyValidator(true)); @Override protected boolean forbidPrivateIndexSettings() { return false; } - private Version version = VersionUtils.randomIndexCompatibleVersion(random()); - static ObjectIntMap expectedDocCountsForGeoHash = null; static ObjectIntMap multiValuedExpectedDocCountsForGeoHash = null; static int numDocs = 100; static String smallestGeoHash = null; - private static IndexRequestBuilder indexCity(String index, String name, List latLon) throws Exception { + private static IndexRequestBuilder indexCity(String index, String name, List points) throws Exception { XContentBuilder source = jsonBuilder().startObject().field("city", name); - if (latLon != null) { - source = source.field("location", latLon); + if (points != null) { + List latLonAsStrings = points.stream().map(ll -> ll.getLat() + ", " + ll.getLon()).collect(Collectors.toList()); + source = source.field("location", latLonAsStrings); + source = source.field("location_as_shape", points.isEmpty() ? null : WKT.toWKT(new MultiPoint(points))); } source = source.endObject(); return client().prepareIndex(index).setSource(source); } - private static IndexRequestBuilder indexCity(String index, String name, String latLon) throws Exception { - return indexCity(index, name, Arrays.asList(latLon)); + private static IndexRequestBuilder indexCity(String index, String name, double lng, double lat) throws Exception { + return indexCity(index, name, Arrays.asList(new Point(lng, lat))); } @Override public void setupSuiteScopeCluster() throws Exception { createIndex("idx_unmapped"); - Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build(); - - assertAcked(prepareCreate("idx").setSettings(settings) - .setMapping("location", "type=geo_point", "city", "type=keyword")); + assertAcked(prepareCreate("idx") + .setMapping("location", "type=geo_point", "location_as_shape", "type=geo_shape", "city", "type=keyword")); + assertAcked(prepareCreate("idx7x", settings(VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0))) + .setMapping("location", "type=geo_point", "location_as_shape", "type=geo_shape", "city", "type=keyword")); List cities = new ArrayList<>(); Random random = random(); @@ -100,7 +107,8 @@ public void setupSuiteScopeCluster() throws Exception { double lng = (360d * random.nextDouble()) - 180d; String randomGeoHash = stringEncode(lng, lat, PRECISION); //Index at the highest resolution - cities.add(indexCity("idx", randomGeoHash, lat + ", " + lng)); + cities.add(indexCity("idx", randomGeoHash, lng, lat)); + cities.add(indexCity("idx7x", randomGeoHash, lng, lat)); expectedDocCountsForGeoHash.put(randomGeoHash, expectedDocCountsForGeoHash.getOrDefault(randomGeoHash, 0) + 1); //Update expected doc counts for all resolutions.. for (int precision = PRECISION - 1; precision > 0; precision--) { @@ -113,19 +121,19 @@ public void setupSuiteScopeCluster() throws Exception { } indexRandom(true, cities); - assertAcked(prepareCreate("multi_valued_idx").setSettings(settings) + assertAcked(prepareCreate("multi_valued_idx").setSettings(settings(VersionUtils.randomIndexCompatibleVersion(random()))) .setMapping("location", "type=geo_point", "city", "type=keyword")); cities = new ArrayList<>(); multiValuedExpectedDocCountsForGeoHash = new ObjectIntHashMap<>(numDocs * 2); for (int i = 0; i < numDocs; i++) { final int numPoints = random.nextInt(4); - List points = new ArrayList<>(); + List points = new ArrayList<>(); Set geoHashes = new HashSet<>(); for (int j = 0; j < numPoints; ++j) { double lat = (180d * random.nextDouble()) - 90d; double lng = (360d * random.nextDouble()) - 180d; - points.add(lat + "," + lng); + points.add(new Point(lng, lat)); // Update expected doc counts for all resolutions.. for (int precision = PRECISION; precision > 0; precision--) { final String geoHash = stringEncode(lng, lat, precision); @@ -142,6 +150,42 @@ public void setupSuiteScopeCluster() throws Exception { ensureSearchable(); } + public void test7xIndexOnly() { + SearchPhaseExecutionException exception = expectThrows(SearchPhaseExecutionException.class, () -> client().prepareSearch("idx7x") + .addAggregation(geohashGrid("aggName").field("location_as_shape")) + .get()); + assertNotNull(exception.getRootCause()); + assertThat(exception.getRootCause().getMessage(), + equalTo("Can't load fielddata on [location_as_shape] because fielddata is unsupported on fields of type [geo_shape]." + + " Use doc values instead.")); + } + + public void test7xIndexWith8Index() { + int precision = randomIntBetween(1, PRECISION); + SearchResponse response = client().prepareSearch("idx", "idx7x") + .addAggregation(geohashGrid("aggName").field("location_as_shape").precision(precision)) + .get(); + assertThat(response.status(), equalTo(RestStatus.OK)); + assertThat(response.getSuccessfulShards(), lessThan(response.getTotalShards())); + GeoGrid geoGrid = response.getAggregations().get("aggName"); + List buckets = geoGrid.getBuckets(); + Object[] propertiesKeys = (Object[]) ((InternalAggregation)geoGrid).getProperty("_key"); + Object[] propertiesDocCounts = (Object[]) ((InternalAggregation)geoGrid).getProperty("_count"); + for (int i = 0; i < buckets.size(); i++) { + GeoGrid.Bucket cell = buckets.get(i); + String geohash = cell.getKeyAsString(); + + long bucketCount = cell.getDocCount(); + int expectedBucketCount = expectedDocCountsForGeoHash.get(geohash); + assertNotSame(bucketCount, 0); + assertEquals("Geohash " + geohash + " has wrong doc count ", + expectedBucketCount, bucketCount); + GeoPoint geoPoint = (GeoPoint) propertiesKeys[i]; + assertThat(stringEncode(geoPoint.lon(), geoPoint.lat(), precision), equalTo(geohash)); + assertThat(propertiesDocCounts[i], equalTo(bucketCount)); + } + } + public void testSimple() throws Exception { for (int precision = 1; precision <= PRECISION; precision++) { SearchResponse response = client().prepareSearch("idx") diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java index 5187be504d3b1..93d125f317f8d 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/AbstractGeoTestCase.java @@ -23,6 +23,7 @@ import com.carrotsearch.hppc.ObjectIntMap; import com.carrotsearch.hppc.ObjectObjectHashMap; import com.carrotsearch.hppc.ObjectObjectMap; +import org.elasticsearch.Version; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.Strings; @@ -41,6 +42,7 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.VersionUtils; import org.elasticsearch.test.geo.RandomGeoGenerator; import java.util.ArrayList; @@ -64,6 +66,7 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase { protected static final String DATELINE_IDX_NAME = "dateline_idx"; protected static final String HIGH_CARD_IDX_NAME = "high_card_idx"; protected static final String IDX_ZERO_NAME = "idx_zero"; + protected static final String IDX_NAME_7x = "idx_7x"; protected static int numDocs; protected static int numUniqueGeoPoints; @@ -74,13 +77,22 @@ public abstract class AbstractGeoTestCase extends ESIntegTestCase { protected static ObjectObjectMap expectedCentroidsForGeoHash = null; protected static final double GEOHASH_TOLERANCE = 1E-5D; + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + @Override public void setupSuiteScopeCluster() throws Exception { createIndex(UNMAPPED_IDX_NAME); assertAcked(prepareCreate(IDX_NAME) - .setMapping(SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point", + .setMapping(SINGLE_VALUED_GEOPOINT_FIELD_NAME, "type=geo_point", SINGLE_VALUED_GEOSHAPE_FIELD_NAME, "type=geo_shape", MULTI_VALUED_FIELD_NAME, "type=geo_point", NUMBER_FIELD_NAME, "type=long", "tag", "type=keyword")); + assertAcked(prepareCreate(IDX_NAME_7x) + .setSettings(settings(VersionUtils.randomPreviousCompatibleVersion(random(), Version.V_8_0_0))) + .setMapping(SINGLE_VALUED_GEOSHAPE_FIELD_NAME, "type=geo_shape")); + singleTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); singleBottomRight = new GeoPoint(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY); multiTopLeft = new GeoPoint(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY); @@ -119,9 +131,16 @@ public void setupSuiteScopeCluster() throws Exception { singleVal = singleValues[i % numUniqueGeoPoints]; multiVal[0] = multiValues[i % numUniqueGeoPoints]; multiVal[1] = multiValues[(i+1) % numUniqueGeoPoints]; + String singleValWKT = "POINT(" + singleVal.lon() + " " + singleVal.lat() + ")"; + builders.add(client().prepareIndex(IDX_NAME_7x).setSource(jsonBuilder() + .startObject() + .field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME, singleValWKT) + .endObject() + )); builders.add(client().prepareIndex(IDX_NAME).setSource(jsonBuilder() .startObject() .array(SINGLE_VALUED_GEOPOINT_FIELD_NAME, singleVal.lon(), singleVal.lat()) + .field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME, singleValWKT) .startArray(MULTI_VALUED_FIELD_NAME) .startArray().value(multiVal[0].lon()).value(multiVal[0].lat()).endArray() .startArray().value(multiVal[1].lon()).value(multiVal[1].lat()).endArray() diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java index 4c980d185fd23..6cb1033bddbaf 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoBoundsIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.GeoPoint; @@ -43,6 +44,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; @@ -52,6 +54,30 @@ public class GeoBoundsIT extends AbstractGeoTestCase { private static final String geoPointAggName = "geoPointBounds"; private static final String geoShapeAggName = "geoShapeBounds"; + public void test7xIndexOnly() { + SearchPhaseExecutionException exception = expectThrows(SearchPhaseExecutionException.class, + () -> client().prepareSearch(IDX_NAME_7x) .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME)) + .get()); + assertNotNull(exception.getRootCause()); + assertThat(exception.getRootCause().getMessage(), + equalTo("Can't load fielddata on [geoshape_value] because fielddata is unsupported on fields of type [geo_shape]." + + " Use doc values instead.")); + } + + public void test7xIndexWith8Index() { + SearchResponse response = client().prepareSearch(IDX_NAME_7x, IDX_NAME) + .addAggregation(geoBounds(geoShapeAggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME).wrapLongitude(false)).get(); + assertThat(response.status(), equalTo(RestStatus.OK)); + assertThat(response.getSuccessfulShards(), lessThan(response.getTotalShards())); + GeoBounds geoBounds = response.getAggregations().get(geoShapeAggName); + GeoPoint topLeft = geoBounds.topLeft(); + GeoPoint bottomRight = geoBounds.bottomRight(); + assertThat(topLeft.lat(), closeTo(singleTopLeft.lat(), GEOHASH_TOLERANCE)); + assertThat(topLeft.lon(), closeTo(singleTopLeft.lon(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lat(), closeTo(singleBottomRight.lat(), GEOHASH_TOLERANCE)); + assertThat(bottomRight.lon(), closeTo(singleBottomRight.lon(), GEOHASH_TOLERANCE)); + } + public void testSingleValuedField() throws Exception { SearchResponse response = client().prepareSearch(IDX_NAME) .addAggregation(geoBounds(geoPointAggName).field(SINGLE_VALUED_GEOPOINT_FIELD_NAME) diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java index 576c4a34e347c..bb706872c134b 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidIT.java @@ -19,8 +19,10 @@ package org.elasticsearch.search.aggregations.metrics; +import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGrid; import org.elasticsearch.search.aggregations.bucket.global.Global; @@ -35,6 +37,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.lessThan; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.sameInstance; @@ -45,6 +48,31 @@ public class GeoCentroidIT extends AbstractGeoTestCase { private static final String aggName = "geoCentroid"; + public void test7xIndexOnly() { + SearchPhaseExecutionException exception = expectThrows(SearchPhaseExecutionException.class, + () -> client().prepareSearch(IDX_NAME_7x) .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME)) + .get()); + assertNotNull(exception.getRootCause()); + assertThat(exception.getRootCause().getMessage(), + equalTo("Can't load fielddata on [geoshape_value] because fielddata is unsupported on fields of type [geo_shape]." + + " Use doc values instead.")); + } + + public void test7xIndexWith8Index() { + SearchResponse response = client().prepareSearch(IDX_NAME_7x, IDX_NAME) + .addAggregation(geoCentroid(aggName).field(SINGLE_VALUED_GEOSHAPE_FIELD_NAME)) + .get(); + assertThat(response.status(), equalTo(RestStatus.OK)); + assertThat(response.getSuccessfulShards(), lessThan(response.getTotalShards())); + GeoCentroid geoCentroid = response.getAggregations().get(aggName); + assertThat(geoCentroid, notNullValue()); + assertThat(geoCentroid.getName(), equalTo(aggName)); + GeoPoint centroid = geoCentroid.centroid(); + assertThat(centroid.lat(), closeTo(singleCentroid.lat(), GEOHASH_TOLERANCE)); + assertThat(centroid.lon(), closeTo(singleCentroid.lon(), GEOHASH_TOLERANCE)); + assertEquals(numDocs, geoCentroid.count()); + } + public void testEmptyAggregation() throws Exception { SearchResponse response = client().prepareSearch(EMPTY_IDX_NAME) .setQuery(matchAllQuery())