diff --git a/server/src/main/java/org/elasticsearch/common/geo/GeoFormatterFactory.java b/server/src/main/java/org/elasticsearch/common/geo/GeoFormatterFactory.java new file mode 100644 index 0000000000000..fae7f3f804524 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/geo/GeoFormatterFactory.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.common.geo; + +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; + +import java.util.List; +import java.util.Locale; +import java.util.function.Function; + +/** + * Output formatters for geo fields. Adds support for vector tiles. + */ +public class GeoFormatterFactory { + + @FunctionalInterface + public interface VectorTileEngine { + /** + * Returns a formatter for a specific tile. + */ + Function, List> getFormatter(int z, int x, int y, int extent); + } + + private static final String MVT = "mvt"; + + /** + * Returns a formatter by name + */ + public static Function, List> getFormatter(String format, Function toGeometry, + VectorTileEngine mvt) { + final int start = format.indexOf('('); + if (start == -1) { + return GeometryFormatterFactory.getFormatter(format, toGeometry); + } + final String formatName = format.substring(0, start); + if (MVT.equals(formatName) == false) { + throw new IllegalArgumentException("Invalid format: " + formatName); + } + final String param = format.substring(start + 1, format.length() - 1); + // we expect either z/x/y or z/x/y@extent + final String[] parts = param.split("@", 3); + if (parts.length > 2) { + throw new IllegalArgumentException( + "Invalid mvt formatter parameter [" + param + "]. Must have the form \"zoom/x/y\" or \"zoom/x/y@extent\"." + ); + } + final int extent = parts.length == 2 ? Integer.parseInt(parts[1]) : 4096; + final String[] tileBits = parts[0].split("/", 4); + if (tileBits.length != 3) { + throw new IllegalArgumentException( + "Invalid tile string [" + parts[0] + "]. Must be three integers in a form \"zoom/x/y\"." + ); + } + final int z = GeoTileUtils.checkPrecisionRange(Integer.parseInt(tileBits[0])); + final int tiles = 1 << z; + final int x = Integer.parseInt(tileBits[1]); + final int y = Integer.parseInt(tileBits[2]); + if (x < 0 || y < 0 || x >= tiles || y >= tiles) { + throw new IllegalArgumentException(String.format(Locale.ROOT, "Zoom/X/Y combination is not valid: %d/%d/%d", z, x, y)); + } + return mvt.getFormatter(z, x, y, extent); + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java b/server/src/main/java/org/elasticsearch/common/geo/SimpleFeatureFactory.java similarity index 88% rename from x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java rename to server/src/main/java/org/elasticsearch/common/geo/SimpleFeatureFactory.java index 5d65213b98f48..a72ee2ec12d5c 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java +++ b/server/src/main/java/org/elasticsearch/common/geo/SimpleFeatureFactory.java @@ -1,26 +1,26 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.vectortile.feature; +package org.elasticsearch.common.geo; import org.apache.lucene.util.BitUtil; import org.elasticsearch.common.geo.SphericalMercatorUtils; import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Comparator; import java.util.List; /** - * Similar to {@link FeatureFactory} but only supports points and rectangles. It is just - * more efficient for those shapes and it does not use external dependencies. + * Transforms points and rectangles objects in WGS84 into mvt features. */ public class SimpleFeatureFactory { @@ -64,11 +64,11 @@ public byte[] point(double lon, double lat) throws IOException { /** * Returns a {@code byte[]} containing the mvt representation of the provided points */ - public byte[] points(List multiPoint) throws IOException { - multiPoint.sort(Comparator.comparingDouble(Point::getLon).thenComparingDouble(Point::getLat)); + public byte[] points(List multiPoint) { + multiPoint.sort(Comparator.comparingDouble(GeoPoint::getLon).thenComparingDouble(GeoPoint::getLat)); final int[] commands = new int[2 * multiPoint.size() + 1]; int pos = 1, prevLon = 0, prevLat = 0, numPoints = 0; - for (Point point : multiPoint) { + for (GeoPoint point : multiPoint) { final int posLon = lon(point.getLon()); if (posLon > extent || posLon < 0) { continue; @@ -90,7 +90,11 @@ public byte[] points(List multiPoint) throws IOException { return EMPTY; } commands[0] = encodeCommand(MOVETO, numPoints); - return writeCommands(commands, 1, pos); + try { + return writeCommands(commands, 1, pos); + } catch (IOException ioe) { + throw new UncheckedIOException(ioe); + } } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index e53e738a6df4e..943c3f860edd5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -19,11 +19,13 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.geo.GeoFormatterFactory; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoShapeUtils; import org.elasticsearch.common.geo.GeoUtils; import org.elasticsearch.common.geo.GeometryFormatterFactory; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.geo.SimpleFeatureFactory; import org.elasticsearch.common.unit.DistanceUnit; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.support.MapXContentParser; @@ -243,7 +245,11 @@ public String typeName() { @Override protected Function, List> getFormatter(String format) { - return GeometryFormatterFactory.getFormatter(format, p -> new Point(p.lon(), p.lat())); + return GeoFormatterFactory.getFormatter(format, p -> new Point(p.getLon(), p.getLat()), + (z, x, y, extent) -> { + final SimpleFeatureFactory featureFactory = new SimpleFeatureFactory(z, x, y, extent); + return points -> org.elasticsearch.core.List.of(featureFactory.points(points)); + }); } @Override diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactoryTests.java b/server/src/test/java/org/elasticsearch/common/geo/SimpleFeatureFactoryTests.java similarity index 87% rename from x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactoryTests.java rename to server/src/test/java/org/elasticsearch/common/geo/SimpleFeatureFactoryTests.java index 9a56dd60edbf6..e172c6ad8702b 100644 --- a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactoryTests.java +++ b/server/src/test/java/org/elasticsearch/common/geo/SimpleFeatureFactoryTests.java @@ -1,14 +1,14 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ -package org.elasticsearch.xpack.vectortile.feature; +package org.elasticsearch.common.geo; import org.apache.lucene.geo.GeoTestUtil; -import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.ESTestCase; @@ -48,7 +48,7 @@ public void testPoint() throws IOException { } } - public void testMultiPoint() throws IOException { + public void testMultiPoint() { int z = randomIntBetween(3, 10); int x = randomIntBetween(0, (1 << z) - 1); int y = randomIntBetween(0, (1 << z) - 1); @@ -57,12 +57,12 @@ public void testMultiPoint() throws IOException { Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); int numPoints = randomIntBetween(2, 10); { - List points = new ArrayList<>(); + List points = new ArrayList<>(); double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() >= l || rectangle.getMaxY() <= l, GeoTestUtil::nextLatitude); double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() >= l || rectangle.getMaxX() <= l, GeoTestUtil::nextLongitude); - points.add(new Point(lon, lat)); + points.add(new GeoPoint(lat, lon)); for (int i = 0; i < numPoints - 1; i++) { - points.add(new Point(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude())); + points.add(new GeoPoint(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude())); } assertThat(builder.points(points).length, Matchers.greaterThan(0)); } @@ -70,7 +70,7 @@ public void testMultiPoint() throws IOException { int xNew = randomValueOtherThanMany(v -> Math.abs(v - x) < 2, () -> randomIntBetween(0, (1 << z) - 1)); int yNew = randomValueOtherThanMany(v -> Math.abs(v - y) < 2, () -> randomIntBetween(0, (1 << z) - 1)); Rectangle rectangleNew = GeoTileUtils.toBoundingBox(xNew, yNew, z); - List points = new ArrayList<>(); + List points = new ArrayList<>(); for (int i = 0; i < numPoints; i++) { double lat = randomValueOtherThanMany( (l) -> rectangleNew.getMinY() >= l || rectangleNew.getMaxY() <= l, @@ -80,7 +80,7 @@ public void testMultiPoint() throws IOException { (l) -> rectangleNew.getMinX() >= l || rectangleNew.getMaxX() <= l, GeoTestUtil::nextLongitude ); - points.add(new Point(lon, lat)); + points.add(new GeoPoint(lat, lon)); } assertThat(builder.points(points).length, Matchers.equalTo(0)); } @@ -95,24 +95,24 @@ public void testPointsMethodConsistency() throws IOException { Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); int extraPoints = randomIntBetween(1, 10); { - List points = new ArrayList<>(); + List points = new ArrayList<>(); double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() > l || rectangle.getMaxY() < l, GeoTestUtil::nextLatitude); double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() > l || rectangle.getMaxX() < l, GeoTestUtil::nextLongitude); - points.add(new Point(lon, lat)); + points.add(new GeoPoint(lat, lon)); assertArrayEquals(builder.points(points), builder.point(lon, lat)); for (int i = 0; i < extraPoints; i++) { - points.add(new Point(lon, lat)); + points.add(new GeoPoint(lat, lon)); } assertArrayEquals(builder.points(points), builder.point(lon, lat)); } { - List points = new ArrayList<>(); + List points = new ArrayList<>(); double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() <= l && rectangle.getMaxY() >= l, GeoTestUtil::nextLatitude); double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() <= l && rectangle.getMaxX() >= l, GeoTestUtil::nextLongitude); - points.add(new Point(lon, lat)); + points.add(new GeoPoint(lat, lon)); assertArrayEquals(builder.points(points), builder.point(lon, lat)); for (int i = 0; i < extraPoints; i++) { - points.add(new Point(lon, lat)); + points.add(new GeoPoint(lat, lon)); } assertArrayEquals(builder.points(points), builder.point(lon, lat)); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java index b4a9485be3986..2880e233be10f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java @@ -8,12 +8,18 @@ package org.elasticsearch.index.mapper; +import org.apache.lucene.geo.GeoTestUtil; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.SimpleFeatureFactory; import org.elasticsearch.script.ScriptCompiler; +import org.hamcrest.Matchers; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; public class GeoPointFieldTypeTests extends FieldTypeTestCase { @@ -60,4 +66,36 @@ public void testFetchSourceValue() throws IOException { sourceValue = "malformed"; assertEquals(Collections.emptyList(), fetchSourceValue(mapper, sourceValue, null)); } + + public void testFetchVectorTile() throws IOException { + MappedFieldType mapper + = new GeoPointFieldMapper.Builder("field", ScriptCompiler.NONE, false).build(new ContentPath()).fieldType(); + final int z = randomIntBetween(1, 10); + int x = randomIntBetween(0, (1 << z) - 1); + int y = randomIntBetween(0, (1 << z) - 1); + final SimpleFeatureFactory featureFactory; + final String mvtString; + if (randomBoolean()) { + int extent = randomIntBetween(1 << 8, 1 << 14); + mvtString = "mvt(" + z + "/" + x + "/" + y + "@" + extent + ")"; + featureFactory = new SimpleFeatureFactory(z, x, y, extent); + } else { + mvtString = "mvt(" + z + "/" + x + "/" + y + ")"; + featureFactory = new SimpleFeatureFactory(z, x, y, 4096); + } + List geoPoints = new ArrayList<>(); + List> values = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(1, 10); i++) { + final double lat = GeoTestUtil.nextLatitude(); + final double lon = GeoTestUtil.nextLongitude(); + List sourceValue = fetchSourceValue(mapper, org.elasticsearch.core.List.of(lon, lat), mvtString); + assertThat(sourceValue.size(), Matchers.equalTo(1)); + assertThat(sourceValue.get(0), Matchers.equalTo(featureFactory.point(lon, lat))); + geoPoints.add(new GeoPoint(lat, lon)); + values.add(org.elasticsearch.core.List.of(lon, lat)); + } + List sourceValue = fetchSourceValue(mapper, values, mvtString); + assertThat(sourceValue.size(), Matchers.equalTo(1)); + assertThat(sourceValue.get(0), Matchers.equalTo(featureFactory.points(geoPoints))); + } } diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 0c452a8b77e02..18b93b8caa8f2 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -12,6 +12,7 @@ esplugin { dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) + testImplementation project(path: xpackModule('vector-tile')) yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) api project(path: ':modules:geo') restTestConfig project(path: ':modules:geo', configuration: 'restTests') diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 970eb537d9a29..2cc5f68c9b511 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.spatial; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.inject.Module; @@ -16,6 +17,7 @@ import org.elasticsearch.license.LicenseUtils; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.ExtensiblePlugin; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.SearchPlugin; @@ -60,7 +62,7 @@ import static java.util.Collections.singletonList; -public class SpatialPlugin extends GeoPlugin implements MapperPlugin, ActionPlugin, SearchPlugin, IngestPlugin { +public class SpatialPlugin extends GeoPlugin implements MapperPlugin, ActionPlugin, SearchPlugin, IngestPlugin, ExtensiblePlugin { private final SpatialUsage usage = new SpatialUsage(); public Collection createGuiceModules() { @@ -74,6 +76,9 @@ protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } + // register the vector tile factory from a different module + private final SetOnce vectorTileExtension = new SetOnce<>(); + @Override public List> getActions() { return singletonList(new ActionPlugin.ActionHandler<>(SpatialStatsAction.INSTANCE, SpatialStatsTransportAction.class)); @@ -84,7 +89,8 @@ public Map getMappers() { Map mappers = new HashMap<>(super.getMappers()); mappers.put(ShapeFieldMapper.CONTENT_TYPE, ShapeFieldMapper.PARSER); mappers.put(PointFieldMapper.CONTENT_TYPE, PointFieldMapper.PARSER); - mappers.put(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, GeoShapeWithDocValuesFieldMapper.PARSER); + mappers.put(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, + new GeoShapeWithDocValuesFieldMapper.TypeParser(vectorTileExtension.get())); return Collections.unmodifiableMap(mappers); } @@ -205,4 +211,10 @@ private ContextParser checkLicense(ContextParser realP return realParser.parse(parser, name); }; } + + @Override + public void loadExtensions(ExtensionLoader loader) { + // we only expect one vector tile extension that comes from the vector tile module. + loader.loadExtensions(VectorTileExtension.class).forEach(vectorTileExtension::set); + } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/VectorTileExtension.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/VectorTileExtension.java new file mode 100644 index 0000000000000..8fb6331f6cbe4 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/VectorTileExtension.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial; + +import org.elasticsearch.common.geo.GeoFormatterFactory; +import org.elasticsearch.geometry.Geometry; + + +public interface VectorTileExtension { + /** + * Get the vector tile engine. This is called when user ask for the MVT format on the field API. + * We are only expecting one instance of a vector tile engine coming from the vector tile module. + */ + GeoFormatterFactory.VectorTileEngine getVectorTileEngine(); +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 407785a4d22bd..a63c90adb1109 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -15,8 +15,8 @@ import org.apache.lucene.search.Query; import org.elasticsearch.Version; import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.geo.GeoFormatterFactory; import org.elasticsearch.common.geo.GeoShapeUtils; -import org.elasticsearch.common.geo.GeometryFormatterFactory; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.Orientation; @@ -35,9 +35,12 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.DocumentParserContext; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xpack.spatial.VectorTileExtension; import org.elasticsearch.xpack.spatial.index.fielddata.plain.AbstractLatLonShapeIndexFieldData; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; @@ -92,10 +95,13 @@ public static class Builder extends FieldMapper.Builder { final Parameter> meta = Parameter.metaParam(); private final Version version; + private final VectorTileExtension vectorTileExtension; - public Builder(String name, Version version, boolean ignoreMalformedByDefault, boolean coerceByDefault) { + public Builder(String name, Version version, boolean ignoreMalformedByDefault, boolean coerceByDefault, + VectorTileExtension vectorTileExtension) { super(name); this.version = version; + this.vectorTileExtension = vectorTileExtension; this.ignoreMalformed = ignoreMalformedParam(m -> builder(m).ignoreMalformed.get(), ignoreMalformedByDefault); this.coerce = coerceParam(m -> builder(m).coerce.get(), coerceByDefault); this.hasDocValues @@ -127,6 +133,7 @@ public GeoShapeWithDocValuesFieldMapper build(ContentPath contentPath) { hasDocValues.get(), orientation.get().value(), parser, + vectorTileExtension, meta.get()); return new GeoShapeWithDocValuesFieldMapper(name, ft, multiFieldsBuilder.build(this, contentPath), copyTo.build(), @@ -137,9 +144,13 @@ public GeoShapeWithDocValuesFieldMapper build(ContentPath contentPath) { public static final class GeoShapeWithDocValuesFieldType extends AbstractShapeGeometryFieldType implements GeoShapeQueryable { + private final VectorTileExtension vectorTileExtension; + public GeoShapeWithDocValuesFieldType(String name, boolean indexed, boolean hasDocValues, - Orientation orientation, GeoShapeParser parser, Map meta) { + Orientation orientation, GeoShapeParser parser, + VectorTileExtension vectorTileExtension, Map meta) { super(name, indexed, false, hasDocValues, parser, orientation, meta); + this.vectorTileExtension = vectorTileExtension; } public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { @@ -173,31 +184,49 @@ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relat @Override protected Function, List> getFormatter(String format) { - return GeometryFormatterFactory.getFormatter(format, Function.identity()); + return GeoFormatterFactory.getFormatter(format, Function.identity(), + (z, x, y, extent) -> { + if (vectorTileExtension == null) { + throw new IllegalArgumentException("vector tile format is not supported"); + } + return vectorTileExtension.getVectorTileEngine().getFormatter(z, x, y, extent); + }); } } - @SuppressWarnings("deprecation") - public static Mapper.TypeParser PARSER = (name, node, parserContext) -> { - FieldMapper.Builder builder; - boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(parserContext.getSettings()); - boolean coerceByDefault = COERCE_SETTING.get(parserContext.getSettings()); - if (LegacyGeoShapeFieldMapper.containsDeprecatedParameter(node.keySet())) { - builder = new LegacyGeoShapeFieldMapper.Builder( - name, - parserContext.indexVersionCreated(), - ignoreMalformedByDefault, - coerceByDefault); - } else { - builder = new GeoShapeWithDocValuesFieldMapper.Builder( - name, - parserContext.indexVersionCreated(), - ignoreMalformedByDefault, - coerceByDefault); + public static class TypeParser implements Mapper.TypeParser { + + private final VectorTileExtension vectorTileExtension; + + public TypeParser(VectorTileExtension vectorTileExtension) { + this.vectorTileExtension = vectorTileExtension; + } + + @Override + @SuppressWarnings("deprecation") + public Mapper.Builder parse(String name, Map node, MappingParserContext parserContext) + throws MapperParsingException { + FieldMapper.Builder builder; + boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(parserContext.getSettings()); + boolean coerceByDefault = COERCE_SETTING.get(parserContext.getSettings()); + if (LegacyGeoShapeFieldMapper.containsDeprecatedParameter(node.keySet())) { + builder = new LegacyGeoShapeFieldMapper.Builder( + name, + parserContext.indexVersionCreated(), + ignoreMalformedByDefault, + coerceByDefault); + } else { + builder = new GeoShapeWithDocValuesFieldMapper.Builder( + name, + parserContext.indexVersionCreated(), + ignoreMalformedByDefault, + coerceByDefault, + vectorTileExtension); + } + builder.parse(name, parserContext, node); + return builder; } - builder.parse(name, parserContext, node); - return builder; - }; + } private final Builder builder; private final GeoShapeIndexer indexer; @@ -246,7 +275,8 @@ public FieldMapper.Builder getMergeBuilder() { simpleName(), builder.version, builder.ignoreMalformed.getDefaultValue().value(), - builder.coerce.getDefaultValue().value() + builder.coerce.getDefaultValue().value(), + builder.vectorTileExtension ).init(this); } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldTypeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldTypeTests.java new file mode 100644 index 0000000000000..76cfb6247ddbb --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldTypeTests.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.index.mapper; + +import org.elasticsearch.Version; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.GeoShapeFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.xpack.vectortile.SpatialVectorTileExtension; +import org.elasticsearch.xpack.vectortile.feature.FeatureFactory; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +public class GeoShapeWithDocValuesFieldTypeTests extends FieldTypeTestCase { + + public void testFetchSourceValue() throws IOException { + MappedFieldType mapper + = new GeoShapeFieldMapper.Builder("field", true, true).build(new ContentPath()).fieldType(); + + Map jsonLineString = org.elasticsearch.core.Map.of("type", "LineString", "coordinates", + org.elasticsearch.core.List.of(org.elasticsearch.core.List.of(42.0, 27.1), org.elasticsearch.core.List.of(30.0, 50.0))); + Map jsonPoint = org.elasticsearch.core.Map.of("type", "Point", "coordinates", + org.elasticsearch.core.List.of(14.0, 15.0)); + Map jsonMalformed = org.elasticsearch.core.Map.of("type", "Point", "coordinates", "foo"); + String wktLineString = "LINESTRING (42.0 27.1, 30.0 50.0)"; + String wktPoint = "POINT (14.0 15.0)"; + String wktMalformed = "POINT foo"; + + // Test a single shape in geojson format. + Object sourceValue = jsonLineString; + assertEquals(org.elasticsearch.core.List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(org.elasticsearch.core.List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a malformed single shape in geojson format + sourceValue = jsonMalformed; + assertEquals(org.elasticsearch.core.List.of(), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(org.elasticsearch.core.List.of(), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes in geojson format. + sourceValue = org.elasticsearch.core.List.of(jsonLineString, jsonPoint); + assertEquals(org.elasticsearch.core.List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(org.elasticsearch.core.List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes including one malformed in geojson format + sourceValue = org.elasticsearch.core.List.of(jsonLineString, jsonMalformed, jsonPoint); + assertEquals(org.elasticsearch.core.List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(org.elasticsearch.core.List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single shape in wkt format. + sourceValue = wktLineString; + assertEquals(org.elasticsearch.core.List.of(jsonLineString), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(org.elasticsearch.core.List.of(wktLineString), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a single malformed shape in wkt format + sourceValue = wktMalformed; + assertEquals(org.elasticsearch.core.List.of(), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(org.elasticsearch.core.List.of(), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes in wkt format. + sourceValue = org.elasticsearch.core.List.of(wktLineString, wktPoint); + assertEquals(org.elasticsearch.core.List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(org.elasticsearch.core.List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + + // Test a list of shapes including one malformed in wkt format + sourceValue = org.elasticsearch.core.List.of(wktLineString, wktMalformed, wktPoint); + assertEquals(org.elasticsearch.core.List.of(jsonLineString, jsonPoint), fetchSourceValue(mapper, sourceValue, null)); + assertEquals(org.elasticsearch.core.List.of(wktLineString, wktPoint), fetchSourceValue(mapper, sourceValue, "wkt")); + } + + public void testFetchVectorTile() throws IOException { + fetchVectorTile(GeometryTestUtils.randomPoint()); + fetchVectorTile(GeometryTestUtils.randomMultiPoint(false)); + fetchVectorTile(GeometryTestUtils.randomRectangle()); + fetchVectorTile(GeometryTestUtils.randomLine(false)); + fetchVectorTile(GeometryTestUtils.randomMultiLine(false)); + fetchVectorTile(GeometryTestUtils.randomPolygon(false)); + fetchVectorTile(GeometryTestUtils.randomMultiPolygon(false)); + } + + private void fetchVectorTile(Geometry geometry) throws IOException { + final MappedFieldType mapper + = new GeoShapeWithDocValuesFieldMapper.Builder("field", Version.CURRENT, false, false, new SpatialVectorTileExtension()) + .build(new ContentPath()).fieldType(); + final int z = randomIntBetween(1, 10); + int x = randomIntBetween(0, (1 << z) - 1); + int y = randomIntBetween(0, (1 << z) - 1); + final FeatureFactory featureFactory; + final String mvtString; + if (randomBoolean()) { + int extent = randomIntBetween(1 << 8, 1 << 14); + mvtString = "mvt(" + z + "/" + x + "/" + y + "@" + extent + ")"; + featureFactory = new FeatureFactory(z, x, y, extent); + } else { + mvtString = "mvt(" + z + "/" + x + "/" + y + ")"; + featureFactory = new FeatureFactory(z, x, y, 4096); + } + + final List sourceValue = fetchSourceValue(mapper, WellKnownText.toWKT(geometry), mvtString); + final List features = featureFactory.getFeatures(geometry); + assertThat(features.size(), Matchers.equalTo(sourceValue.size())); + for (int i = 0; i < features.size(); i++) { + assertThat(sourceValue.get(i), Matchers.equalTo(features.get(i))); + } + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java index 1da8079a0e2de..b91bcd3094775 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/ingest/CircleProcessorTests.java @@ -213,7 +213,7 @@ public void testGeoShapeQueryAcrossDateline() throws IOException { Geometry geometry = SpatialUtils.createRegularGeoShapePolygon(circle, numSides); GeoShapeWithDocValuesFieldType shapeType - = new GeoShapeWithDocValuesFieldType(fieldName, true, false, Orientation.RIGHT, null, Collections.emptyMap()); + = new GeoShapeWithDocValuesFieldType(fieldName, true, false, Orientation.RIGHT, null, null, Collections.emptyMap()); SearchExecutionContext mockedContext = mock(SearchExecutionContext.class); when(mockedContext.getFieldType(any())).thenReturn(shapeType); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java index 2a931473ba1d5..014407aaeb924 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/bucket/geogrid/GeoShapeGeoGridTestCase.java @@ -294,7 +294,7 @@ private void testCase(Query query, int precision, GeoBoundingBox geoBoundingBox, } MappedFieldType fieldType - = new GeoShapeWithDocValuesFieldType(FIELD_NAME, true, true, Orientation.RIGHT, null, Collections.emptyMap()); + = new GeoShapeWithDocValuesFieldType(FIELD_NAME, true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); aggregator.preCollection(); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java index 434e423cd2d76..2408be6e50a24 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java @@ -58,7 +58,7 @@ public void testEmpty() throws Exception { .wrapLongitude(false); MappedFieldType fieldType - = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, Collections.emptyMap()); + = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); @@ -86,7 +86,7 @@ public void testUnmappedFieldWithDocs() throws Exception { .wrapLongitude(false); MappedFieldType fieldType - = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, Collections.emptyMap()); + = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); @@ -108,7 +108,7 @@ public void testMissing() throws Exception { w.addDocument(doc); MappedFieldType fieldType - = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, Collections.emptyMap()); + = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); Point point = GeometryTestUtils.randomPoint(false); double lon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(point.getX())); @@ -140,7 +140,7 @@ public void testInvalidMissing() throws Exception { w.addDocument(doc); MappedFieldType fieldType - = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, Collections.emptyMap()); + = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); GeoBoundsAggregationBuilder aggBuilder = new GeoBoundsAggregationBuilder("my_agg") .field("field") @@ -200,7 +200,7 @@ public void testRandomShapes() throws Exception { .wrapLongitude(false); MappedFieldType fieldType - = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, Collections.emptyMap()); + = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); InternalGeoBounds bounds = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java index dd52083919008..68d4200aea5cc 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeCentroidAggregatorTests.java @@ -64,7 +64,7 @@ public void testEmpty() throws Exception { .field("field"); MappedFieldType fieldType - = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, Collections.emptyMap()); + = new GeoShapeWithDocValuesFieldType("field", true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); try (IndexReader reader = w.getReader()) { IndexSearcher searcher = new IndexSearcher(reader); InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); @@ -87,12 +87,12 @@ public void testUnmapped() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MappedFieldType fieldType = new GeoShapeWithDocValuesFieldType("another_field", - true, true, Orientation.RIGHT, null, Collections.emptyMap()); + true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertNull(result.centroid()); fieldType = new GeoShapeWithDocValuesFieldType("field", - true, true, Orientation.RIGHT, null, Collections.emptyMap()); + true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertNull(result.centroid()); assertFalse(AggregationInspectionHelper.hasValue(result)); @@ -117,7 +117,7 @@ public void testUnmappedWithMissing() throws Exception { IndexSearcher searcher = new IndexSearcher(reader); MappedFieldType fieldType = new GeoShapeWithDocValuesFieldType("another_field", - true, true, Orientation.RIGHT, null, Collections.emptyMap()); + true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); InternalGeoCentroid result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); assertThat(result.centroid(), equalTo(expectedCentroid)); assertTrue(AggregationInspectionHelper.hasValue(result)); @@ -180,7 +180,7 @@ public void testSingleValuedField() throws Exception { private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) throws IOException { MappedFieldType fieldType = new GeoShapeWithDocValuesFieldType("field", - true, true, Orientation.RIGHT, null, Collections.emptyMap()); + true, true, Orientation.RIGHT, null, null, Collections.emptyMap()); GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg") .field("field"); try (IndexReader reader = w.getReader()) { diff --git a/x-pack/plugin/vector-tile/build.gradle b/x-pack/plugin/vector-tile/build.gradle index 337d4986cbf1d..5cb14d2f82ba7 100644 --- a/x-pack/plugin/vector-tile/build.gradle +++ b/x-pack/plugin/vector-tile/build.gradle @@ -22,11 +22,12 @@ esplugin { name 'vector-tile' description 'A plugin for mapbox vector tile features' classname 'org.elasticsearch.xpack.vectortile.VectorTilePlugin' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['spatial'] } dependencies { compileOnly project(path: xpackModule('core')) + compileOnly project(path: xpackModule('spatial')) testImplementation(testArtifact(project(xpackModule('core')))) api "com.wdtinc:mapbox-vector-tile:3.1.0" api "com.google.protobuf:protobuf-java:3.14.0" diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/SpatialVectorTileExtension.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/SpatialVectorTileExtension.java new file mode 100644 index 0000000000000..261199c24c9ef --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/SpatialVectorTileExtension.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile; + +import org.elasticsearch.common.geo.GeoFormatterFactory; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.xpack.spatial.VectorTileExtension; +import org.elasticsearch.xpack.vectortile.feature.FeatureFactory; + +import java.util.ArrayList; + +/** + * Unique implementation of VectorTileExtension so we can transform geometries + * into its vector tile representation from the spatial module. + */ +public class SpatialVectorTileExtension implements VectorTileExtension { + + @Override + public GeoFormatterFactory.VectorTileEngine getVectorTileEngine() { + return (z, x, y, extent) -> { + final FeatureFactory featureFactory = new FeatureFactory(z, x, y, extent); + return geometries -> { + final Geometry geometry = (geometries.size() == 1) ? geometries.get(0) : new GeometryCollection<>(geometries); + return new ArrayList<>(featureFactory.getFeatures(geometry)); + }; + }; + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java index 223fcc601edbd..0fab0ce78f4b7 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java @@ -12,6 +12,7 @@ import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; import com.wdtinc.mapbox_vector_tile.adapt.jts.JtsAdapter; import com.wdtinc.mapbox_vector_tile.adapt.jts.TileGeomResult; +import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import com.wdtinc.mapbox_vector_tile.build.MvtLayerParams; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; @@ -34,6 +35,7 @@ import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.LineString; +import java.util.ArrayList; import java.util.List; /** @@ -42,6 +44,7 @@ public class FeatureFactory { private final IGeometryFilter acceptAllGeomFilter = geometry -> true; + private final IUserDataConverter userDataIgnoreConverter = new UserDataIgnoreConverter(); private final MvtLayerParams layerParams; private final GeometryFactory geomFactory = new GeometryFactory(); private final MvtLayerProps layerProps = new MvtLayerProps(); @@ -60,7 +63,7 @@ public FeatureFactory(int z, int x, int y, int extent) { this.layerParams = new MvtLayerParams(extent, extent); } - public List getFeatures(Geometry geometry, IUserDataConverter userData) { + public List getFeatures(Geometry geometry) { final TileGeomResult tileGeom = JtsAdapter.createTileGeom( JtsAdapter.flatFeatureList(geometry.visit(builder)), tileEnvelope, @@ -70,11 +73,10 @@ public List getFeatures(Geometry geometry, IUserDataCon acceptAllGeomFilter ); // MVT tile geometry to MVT features - return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, userData); - } - - public MvtLayerProps getLayerProps() { - return layerProps; + final List features = JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, userDataIgnoreConverter); + final List byteFeatures = new ArrayList<>(features.size()); + features.forEach(f -> byteFeatures.add(f.toByteArray())); + return byteFeatures; } private static class JTSGeometryBuilder implements GeometryVisitor { diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java index 001633f7d47f9..6e5da2a909cb7 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.vectortile.rest; import com.wdtinc.mapbox_vector_tile.VectorTile; -import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -17,10 +16,9 @@ import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.common.geo.SimpleFeatureFactory; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStream; -import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; @@ -49,8 +47,6 @@ import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.profile.SearchProfileShardResults; import org.elasticsearch.search.sort.SortBuilder; -import org.elasticsearch.xpack.vectortile.feature.FeatureFactory; -import org.elasticsearch.xpack.vectortile.feature.SimpleFeatureFactory; import java.io.IOException; import java.util.Collection; @@ -176,8 +172,12 @@ private static SearchRequestBuilder searchRequestBuilder(RestCancellableNodeClie final SearchRequestBuilder searchRequestBuilder = client.prepareSearch(request.getIndexes()); searchRequestBuilder.setSize(request.getSize()); searchRequestBuilder.setFetchSource(false); - // TODO: I wonder if we can leverage field and format so what we get in the result is already the mvt commands. - searchRequestBuilder.addFetchField(new FieldAndFormat(request.getField(), null)); + searchRequestBuilder.addFetchField( + new FieldAndFormat( + request.getField(), + "mvt(" + request.getZ() + "/" + request.getX() + "/" + request.getY() + "@" + request.getExtent() + ")" + ) + ); for (FieldAndFormat field : request.getFieldAndFormats()) { searchRequestBuilder.addFetchField(field); } @@ -225,13 +225,20 @@ private static SearchRequestBuilder searchRequestBuilder(RestCancellableNodeClie return searchRequestBuilder; } - private static VectorTile.Tile.Layer.Builder buildHitsLayer(SearchHit[] hits, VectorTileRequest request) { - final FeatureFactory featureFactory = new FeatureFactory(request.getZ(), request.getX(), request.getY(), request.getExtent()); - final GeometryParser parser = new GeometryParser(true, false, false); + @SuppressWarnings("unchecked") + private static VectorTile.Tile.Layer.Builder buildHitsLayer(SearchHit[] hits, VectorTileRequest request) throws IOException { final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, request.getExtent()); final List fields = request.getFieldAndFormats(); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); for (SearchHit searchHit : hits) { - final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> { + final DocumentField geoField = searchHit.field(request.getField()); + if (geoField == null) { + continue; + } + for (Object feature : geoField) { + featureBuilder.clear(); + featureBuilder.mergeFrom((byte[]) feature); VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId()); if (fields != null) { for (FieldAndFormat field : fields) { @@ -241,12 +248,10 @@ private static VectorTile.Tile.Layer.Builder buildHitsLayer(SearchHit[] hits, Ve } } } - }; - // TODO: See comment on field formats. - final Geometry geometry = parser.parseGeometry(searchHit.field(request.getField()).getValue()); - hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags)); + hitsLayerBuilder.addFeatures(featureBuilder); + } } - VectorTileUtils.addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); + VectorTileUtils.addPropertiesToLayer(hitsLayerBuilder, layerProps); return hitsLayerBuilder; } diff --git a/x-pack/plugin/vector-tile/src/main/resources/META-INF/services/org.elasticsearch.xpack.spatial.VectorTileExtension b/x-pack/plugin/vector-tile/src/main/resources/META-INF/services/org.elasticsearch.xpack.spatial.VectorTileExtension new file mode 100644 index 0000000000000..5f2a083ea5410 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/resources/META-INF/services/org.elasticsearch.xpack.spatial.VectorTileExtension @@ -0,0 +1,8 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0; you may not use this file except in compliance with the Elastic License +# 2.0. +# + +org.elasticsearch.xpack.vectortile.SpatialVectorTileExtension diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoriesConsistencyTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoriesConsistencyTests.java index d120fe6145452..5ba51d51636d4 100644 --- a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoriesConsistencyTests.java +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoriesConsistencyTests.java @@ -7,10 +7,9 @@ package org.elasticsearch.xpack.vectortile.feature; -import com.wdtinc.mapbox_vector_tile.VectorTile; -import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; - import org.apache.lucene.geo.GeoTestUtil; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.SimpleFeatureFactory; import org.elasticsearch.common.geo.SphericalMercatorUtils; import org.elasticsearch.geometry.MultiPoint; import org.elasticsearch.geometry.Point; @@ -37,21 +36,19 @@ public void testPoint() throws IOException { SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); FeatureFactory factory = new FeatureFactory(z, x, y, extent); List points = new ArrayList<>(); + List geoPoints = new ArrayList<>(); for (int i = 0; i < 10; i++) { double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() > l || rectangle.getMaxY() < l, GeoTestUtil::nextLatitude); double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() > l || rectangle.getMaxX() < l, GeoTestUtil::nextLongitude); byte[] b1 = builder.point(lon, lat); Point point = new Point(lon, lat); - List features = factory.getFeatures(point, new UserDataIgnoreConverter()); - assertThat(features.size(), Matchers.equalTo(1)); - byte[] b2 = features.get(0).toByteArray(); + byte[] b2 = factory.getFeatures(point).get(0); assertArrayEquals(b1, b2); points.add(point); + geoPoints.add(new GeoPoint(lat, lon)); } - byte[] b1 = builder.points(points); - List features = factory.getFeatures(new MultiPoint(points), new UserDataIgnoreConverter()); - assertThat(features.size(), Matchers.equalTo(1)); - byte[] b2 = features.get(0).toByteArray(); + byte[] b1 = builder.points(geoPoints); + byte[] b2 = factory.getFeatures(new MultiPoint(points)).get(0); assertArrayEquals(b1, b2); } @@ -68,9 +65,7 @@ public void testIssue74341() throws IOException { FeatureFactory factory = new FeatureFactory(z, x, y, extent); byte[] b1 = builder.point(lon, lat); Point point = new Point(lon, lat); - List features = factory.getFeatures(point, new UserDataIgnoreConverter()); - assertThat(features.size(), Matchers.equalTo(1)); - byte[] b2 = features.get(0).toByteArray(); + byte[] b2 = factory.getFeatures(point).get(0); assertThat(Arrays.equals(b1, b2), Matchers.equalTo(false)); } @@ -91,9 +86,7 @@ public void testRectangle() throws IOException { Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); for (int i = 0; i < extent; i++) { byte[] b1 = builder.box(r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); - List features = factory.getFeatures(r, new UserDataIgnoreConverter()); - assertThat(features.size(), Matchers.equalTo(1)); - byte[] b2 = features.get(0).toByteArray(); + byte[] b2 = factory.getFeatures(r).get(0); assertArrayEquals(extent + "", b1, b2); } } diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java index 3fd62db0beedf..f5f197ff5c563 100644 --- a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.vectortile.feature; import com.wdtinc.mapbox_vector_tile.VectorTile; -import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import org.apache.lucene.geo.GeoTestUtil; import org.elasticsearch.geometry.Geometry; @@ -25,6 +24,7 @@ import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; +import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -33,56 +33,56 @@ public class FeatureFactoryTests extends ESTestCase { - public void testPoint() { + public void testPoint() throws IOException { doTestGeometry(this::buildPoint, features -> { assertThat(features.size(), Matchers.equalTo(1)); assertThat(features.get(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POINT)); }); } - public void testMultiPoint() { + public void testMultiPoint() throws IOException { doTestGeometry(this::buildMultiPoint, features -> { assertThat(features.size(), Matchers.equalTo(1)); assertThat(features.get(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POINT)); }); } - public void testRectangle() { + public void testRectangle() throws IOException { doTestGeometry(r -> r, features -> { assertThat(features.size(), Matchers.equalTo(1)); assertThat(features.get(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POLYGON)); }); } - public void testLine() { + public void testLine() throws IOException { doTestGeometry(this::buildLine, features -> { assertThat(features.size(), Matchers.equalTo(1)); assertThat(features.get(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.LINESTRING)); }); } - public void testMultiLine() { + public void testMultiLine() throws IOException { doTestGeometry(this::buildMultiLine, features -> { assertThat(features.size(), Matchers.equalTo(1)); assertThat(features.get(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.LINESTRING)); }); } - public void testPolygon() { + public void testPolygon() throws IOException { doTestGeometry(this::buildPolygon, features -> { assertThat(features.size(), Matchers.equalTo(1)); assertThat(features.get(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POLYGON)); }); } - public void testMultiPolygon() { + public void testMultiPolygon() throws IOException { doTestGeometry(this::buildMultiPolygon, features -> { assertThat(features.size(), Matchers.equalTo(1)); assertThat(features.get(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POLYGON)); }); } - public void testGeometryCollection() { + public void testGeometryCollection() throws IOException { doTestGeometry(this::buildGeometryCollection, features -> { assertThat(features.size(), Matchers.equalTo(2)); assertThat(features.get(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.LINESTRING)); @@ -90,7 +90,8 @@ public void testGeometryCollection() { }); } - private void doTestGeometry(Function provider, Consumer> consumer) { + private void doTestGeometry(Function provider, Consumer> consumer) + throws IOException { final int z = randomIntBetween(3, 10); final int x = randomIntBetween(2, (1 << z) - 1); final int y = randomIntBetween(2, (1 << z) - 1); @@ -98,13 +99,17 @@ private void doTestGeometry(Function provider, Consumer features = builder.getFeatures(provider.apply(r), new UserDataIgnoreConverter()); + final List byteFeatures = builder.getFeatures(provider.apply(r)); + final List features = new ArrayList<>(byteFeatures.size()); + for (byte[] byteFeature : byteFeatures) { + features.add(VectorTile.Tile.Feature.parseFrom(byteFeature)); + } consumer.accept(features); } { final Rectangle r = GeoTileUtils.toBoundingBox(x - 2, y, z); - final List features = builder.getFeatures(provider.apply(r), new UserDataIgnoreConverter()); - assertThat(features.size(), Matchers.equalTo(0)); + final List byteFeatures = builder.getFeatures(provider.apply(r)); + assertThat(byteFeatures.size(), Matchers.equalTo(0)); } }