diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorSupplier.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorSupplier.java index 9e3db7f6860e2..1fbc63baac581 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorSupplier.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/GeoCentroidAggregatorSupplier.java @@ -30,6 +30,6 @@ @FunctionalInterface public interface GeoCentroidAggregatorSupplier extends AggregatorSupplier { - GeoCentroidAggregator build(String name, SearchContext context, Aggregator parent, + MetricsAggregator build(String name, SearchContext context, Aggregator parent, ValuesSource valuesSource, Map metadata) throws IOException; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java index ea76610501a91..1bd51f154879c 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/metrics/InternalGeoCentroid.java @@ -53,7 +53,7 @@ public static double decodeLongitude(long encodedLatLon) { return GeoEncodingUtils.decodeLongitude((int) (encodedLatLon & 0xFFFFFFFFL)); } - InternalGeoCentroid(String name, GeoPoint centroid, long count, Map metadata) { + public InternalGeoCentroid(String name, GeoPoint centroid, long count, Map metadata) { super(name, metadata); assert (centroid == null) == (count == 0); this.centroid = centroid; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 920e24bf93b67..c497cbe1c007a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -45,7 +45,8 @@ public enum Feature { SECURITY_TOKEN_SERVICE(OperationMode.GOLD, false), SECURITY_API_KEY_SERVICE(OperationMode.MISSING, false), SECURITY_AUTHORIZATION_REALM(OperationMode.PLATINUM, true), - SECURITY_AUTHORIZATION_ENGINE(OperationMode.PLATINUM, true); + SECURITY_AUTHORIZATION_ENGINE(OperationMode.PLATINUM, true), + SPATIAL_GEO_CENTROID(OperationMode.GOLD, true); final OperationMode minimumOperationMode; final boolean needsActive; diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 8419f0000cc75..e6630238e78f3 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -18,7 +18,7 @@ dependencies { restResources { restApi { - includeCore '_common', 'indices', 'index', 'search' + includeCore '_common', 'bulk', 'indices', 'index', 'search' } restTests { includeCore 'geo_shape' @@ -26,6 +26,7 @@ restResources { } testClusters.integTest { + setting 'xpack.license.self_generated.type', 'trial' testDistribution = 'DEFAULT' } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index e07f65aac30d8..2afa7f199d0c2 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -10,21 +10,27 @@ import org.elasticsearch.geo.GeoPlugin; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.license.LicenseUtils; +import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.SearchPlugin; import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregatorSupplier; +import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregatorSupplier; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; -import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; +import org.elasticsearch.xpack.spatial.aggregations.metrics.GeoShapeCentroidAggregator; import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper; import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder; import org.elasticsearch.xpack.spatial.ingest.CircleProcessor; +import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; @@ -39,6 +45,11 @@ public class SpatialPlugin extends GeoPlugin implements ActionPlugin, MapperPlugin, SearchPlugin, IngestPlugin { + // to be overriden by tests + protected XPackLicenseState getLicenseState() { + return XPackPlugin.getSharedLicenseState(); + } + @Override public List> getActions() { return Arrays.asList( @@ -62,7 +73,7 @@ public List> getQueries() { @Override public List> getAggregationExtentions() { - return List.of(SpatialPlugin::registerGeoShapeBoundsAggregator); + return List.of(this::registerGeoShapeBoundsAggregator, this::registerGeoShapeCentroidAggregator); } @Override @@ -70,10 +81,21 @@ public Map getProcessors(Processor.Parameters paramet return Map.of(CircleProcessor.TYPE, new CircleProcessor.Factory()); } - public static void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) { + public void registerGeoShapeBoundsAggregator(ValuesSourceRegistry.Builder builder) { builder.register(GeoBoundsAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), (GeoBoundsAggregatorSupplier) (name, aggregationContext, parent, valuesSource, wrapLongitude, metadata) -> new GeoShapeBoundsAggregator(name, aggregationContext, parent, (GeoShapeValuesSource) valuesSource, wrapLongitude, metadata)); } + + public void registerGeoShapeCentroidAggregator(ValuesSourceRegistry.Builder builder) { + builder.register(GeoCentroidAggregationBuilder.NAME, GeoShapeValuesSourceType.instance(), + (GeoCentroidAggregatorSupplier) (name, aggregationContext, parent, valuesSource, metadata) + -> { + if (getLicenseState().isAllowed(XPackLicenseState.Feature.SPATIAL_GEO_CENTROID)) { + return new GeoShapeCentroidAggregator(name, aggregationContext, parent, (GeoShapeValuesSource) valuesSource, metadata); + } + throw LicenseUtils.newComplianceException("geo_centroid aggregation on geo_shape fields"); + }); + } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java new file mode 100644 index 0000000000000..d92b1a155c8b9 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregator.java @@ -0,0 +1,156 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +package org.elasticsearch.xpack.spatial.aggregations.metrics; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.ByteArray; +import org.elasticsearch.common.util.DoubleArray; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.metrics.CompensatedSum; +import org.elasticsearch.search.aggregations.metrics.InternalGeoCentroid; +import org.elasticsearch.search.aggregations.metrics.MetricsAggregator; +import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.xpack.spatial.index.fielddata.DimensionalShapeType; +import org.elasticsearch.xpack.spatial.index.fielddata.MultiGeoShapeValues; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; + +import java.io.IOException; +import java.util.Map; + +/** + * A geo metric aggregator that computes a geo-centroid from a {@code geo_shape} type field + */ +public final class GeoShapeCentroidAggregator extends MetricsAggregator { + private final GeoShapeValuesSource valuesSource; + private DoubleArray lonSum, lonCompensations, latSum, latCompensations, weightSum, weightCompensations; + private LongArray counts; + private ByteArray dimensionalShapeTypes; + + public GeoShapeCentroidAggregator(String name, SearchContext context, Aggregator parent, + GeoShapeValuesSource valuesSource, Map metadata) throws IOException { + super(name, context, parent, metadata); + this.valuesSource = valuesSource; + if (valuesSource != null) { + final BigArrays bigArrays = context.bigArrays(); + lonSum = bigArrays.newDoubleArray(1, true); + lonCompensations = bigArrays.newDoubleArray(1, true); + latSum = bigArrays.newDoubleArray(1, true); + latCompensations = bigArrays.newDoubleArray(1, true); + weightSum = bigArrays.newDoubleArray(1, true); + weightCompensations = bigArrays.newDoubleArray(1, true); + counts = bigArrays.newLongArray(1, true); + dimensionalShapeTypes = bigArrays.newByteArray(1, true); + } + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + if (valuesSource == null) { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + final BigArrays bigArrays = context.bigArrays(); + final MultiGeoShapeValues values = valuesSource.geoShapeValues(ctx); + final CompensatedSum compensatedSumLat = new CompensatedSum(0, 0); + final CompensatedSum compensatedSumLon = new CompensatedSum(0, 0); + final CompensatedSum compensatedSumWeight = new CompensatedSum(0, 0); + + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + latSum = bigArrays.grow(latSum, bucket + 1); + lonSum = bigArrays.grow(lonSum, bucket + 1); + weightSum = bigArrays.grow(weightSum, bucket + 1); + lonCompensations = bigArrays.grow(lonCompensations, bucket + 1); + latCompensations = bigArrays.grow(latCompensations, bucket + 1); + weightCompensations = bigArrays.grow(weightCompensations, bucket + 1); + counts = bigArrays.grow(counts, bucket + 1); + dimensionalShapeTypes = bigArrays.grow(dimensionalShapeTypes, bucket + 1); + + if (values.advanceExact(doc)) { + final int valueCount = values.docValueCount(); + // increment by the number of points for this document + counts.increment(bucket, valueCount); + // Compute the sum of double values with Kahan summation algorithm which is more + // accurate than naive summation. + DimensionalShapeType shapeType = DimensionalShapeType.fromOrdinalByte(dimensionalShapeTypes.get(bucket)); + double sumLat = latSum.get(bucket); + double compensationLat = latCompensations.get(bucket); + double sumLon = lonSum.get(bucket); + double compensationLon = lonCompensations.get(bucket); + double sumWeight = weightSum.get(bucket); + double compensatedWeight = weightCompensations.get(bucket); + + compensatedSumLat.reset(sumLat, compensationLat); + compensatedSumLon.reset(sumLon, compensationLon); + compensatedSumWeight.reset(sumWeight, compensatedWeight); + + // update the sum + for (int i = 0; i < valueCount; ++i) { + MultiGeoShapeValues.GeoShapeValue value = values.nextValue(); + int compares = shapeType.compareTo(value.dimensionalShapeType()); + if (compares < 0) { + double coordinateWeight = value.weight(); + compensatedSumLat.reset(coordinateWeight * value.lat(), 0.0); + compensatedSumLon.reset(coordinateWeight * value.lon(), 0.0); + compensatedSumWeight.reset(coordinateWeight, 0.0); + dimensionalShapeTypes.set(bucket, (byte) value.dimensionalShapeType().ordinal()); + } else if (compares == 0) { + double coordinateWeight = value.weight(); + // weighted latitude + compensatedSumLat.add(coordinateWeight * value.lat()); + // weighted longitude + compensatedSumLon.add(coordinateWeight * value.lon()); + // weight + compensatedSumWeight.add(coordinateWeight); + } + // else (compares > 0) + // do not modify centroid calculation since shape is of lower dimension than the running dimension + + } + lonSum.set(bucket, compensatedSumLon.value()); + lonCompensations.set(bucket, compensatedSumLon.delta()); + latSum.set(bucket, compensatedSumLat.value()); + latCompensations.set(bucket, compensatedSumLat.delta()); + weightSum.set(bucket, compensatedSumWeight.value()); + weightCompensations.set(bucket, compensatedSumWeight.delta()); + } + } + }; + } + + @Override + public InternalAggregation buildAggregation(long bucket) { + if (valuesSource == null || bucket >= counts.size()) { + return buildEmptyAggregation(); + } + final long bucketCount = counts.get(bucket); + final double bucketWeight = weightSum.get(bucket); + final GeoPoint bucketCentroid = (bucketWeight > 0) + ? new GeoPoint(latSum.get(bucket) / bucketWeight, lonSum.get(bucket) / bucketWeight) + : null; + return new InternalGeoCentroid(name, bucketCentroid , bucketCount, metadata()); + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalGeoCentroid(name, null, 0L, metadata()); + } + + @Override + public void doClose() { + Releasables.close(latSum, latCompensations, lonSum, lonCompensations, counts, weightSum, weightCompensations, + dimensionalShapeTypes); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/LocalStateSpatialPlugin.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/LocalStateSpatialPlugin.java new file mode 100644 index 0000000000000..42c0769c3bd6b --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/LocalStateSpatialPlugin.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial; + +import org.apache.lucene.util.LuceneTestCase; +import org.elasticsearch.license.License; +import org.elasticsearch.license.TestUtils; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.test.VersionUtils; + +/** + * This class overrides the {@link SpatialPlugin} in order + * to provide the integration test clusters a hook into a real + * {@link XPackLicenseState}. In the cases that this is used, the + * actual license's operation mode is not important + */ +public class LocalStateSpatialPlugin extends SpatialPlugin { + protected XPackLicenseState getLicenseState() { + TestUtils.UpdatableLicenseState licenseState = new TestUtils.UpdatableLicenseState(); + License.OperationMode operationMode = License.OperationMode.TRIAL; + licenseState.update(operationMode, true, VersionUtils.randomVersion(LuceneTestCase.random())); + return licenseState; + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java new file mode 100644 index 0000000000000..85c377c421427 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/SpatialPluginTests.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.spatial; + +import org.elasticsearch.ElasticsearchSecurityException; +import org.elasticsearch.license.License; +import org.elasticsearch.license.TestUtils; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregatorSupplier; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.VersionUtils; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; + +import java.util.List; +import java.util.function.Consumer; + +import static org.hamcrest.Matchers.equalTo; + +public class SpatialPluginTests extends ESTestCase { + + public void testGeoCentroidLicenseCheck() { + for (License.OperationMode operationMode : License.OperationMode.values()) { + SpatialPlugin plugin = getPluginWithOperationMode(operationMode); + ValuesSourceRegistry.Builder registryBuilder = new ValuesSourceRegistry.Builder(); + List> registrar = plugin.getAggregationExtentions(); + registrar.forEach(c -> c.accept(registryBuilder)); + ValuesSourceRegistry registry = registryBuilder.build(); + GeoCentroidAggregatorSupplier centroidSupplier = (GeoCentroidAggregatorSupplier) registry.getAggregator( + GeoShapeValuesSourceType.instance(), GeoCentroidAggregationBuilder.NAME); + if (License.OperationMode.TRIAL != operationMode && + License.OperationMode.compare(operationMode, License.OperationMode.GOLD) < 0) { + ElasticsearchSecurityException exception = expectThrows(ElasticsearchSecurityException.class, + () -> centroidSupplier.build(null, null, null, null, null)); + assertThat(exception.getMessage(), + equalTo("current license is non-compliant for [geo_centroid aggregation on geo_shape fields]")); + } + } + } + + private SpatialPlugin getPluginWithOperationMode(License.OperationMode operationMode) { + return new SpatialPlugin() { + protected XPackLicenseState getLicenseState() { + TestUtils.UpdatableLicenseState licenseState = new TestUtils.UpdatableLicenseState(); + licenseState.update(operationMode, true, VersionUtils.randomVersion(random())); + return licenseState; + } + }; + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java new file mode 100644 index 0000000000000..eeb2c5bc35363 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/aggregations/metrics/GeoShapeCentroidAggregatorTests.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.spatial.aggregations.metrics; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.geo.GeoEncodingUtils; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.metrics.CompensatedSum; +import org.elasticsearch.search.aggregations.metrics.GeoCentroidAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalGeoCentroid; +import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.index.fielddata.CentroidCalculator; +import org.elasticsearch.xpack.spatial.index.fielddata.DimensionalShapeType; +import org.elasticsearch.xpack.spatial.index.mapper.BinaryGeoShapeDocValuesField; +import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; +import org.locationtech.spatial4j.exception.InvalidShapeException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Function; + +import static org.hamcrest.Matchers.equalTo; + +public class GeoShapeCentroidAggregatorTests extends AggregatorTestCase { + + private static final double GEOHASH_TOLERANCE = 1E-6D; + + @Override + protected List getSearchPlugins() { + return List.of(new LocalStateSpatialPlugin()); + } + + public void testEmpty() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg") + .field("field"); + + MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("field"); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertNull(result.centroid()); + assertFalse(AggregationInspectionHelper.hasValue(result)); + } + } + } + + public void testUnmapped() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg") + .field("another_field"); + + Document document = new Document(); + document.add(new LatLonDocValuesField("field", 10, 10)); + w.addDocument(document); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + + MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("another_field"); + InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertNull(result.centroid()); + + fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("field"); + result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertNull(result.centroid()); + assertFalse(AggregationInspectionHelper.hasValue(result)); + } + } + } + + public void testUnmappedWithMissing() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg") + .field("another_field") + .missing("POINT(6.475031 53.69437)"); + + double normalizedLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(53.69437)); + double normalizedLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(6.475031)); + GeoPoint expectedCentroid = new GeoPoint(normalizedLat, normalizedLon); + Document document = new Document(); + document.add(new LatLonDocValuesField("field", 10, 10)); + w.addDocument(document); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + + MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("another_field"); + InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + assertThat(result.centroid(), equalTo(expectedCentroid)); + assertTrue(AggregationInspectionHelper.hasValue(result)); + } + } + } + + @SuppressWarnings("unchecked") + public void testSingleValuedField() throws Exception { + int numDocs = scaledRandomIntBetween(64, 256); + List geometries = new ArrayList<>(); + DimensionalShapeType targetShapeType = DimensionalShapeType.POINT; + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + for (int i = 0; i < numDocs; i++) { + Function geometryGenerator = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPoint, + GeometryTestUtils::randomMultiPolygon + ); + Geometry geometry = geometryGenerator.apply(false); + try { + geometries.add(indexer.prepareForIndexing(geometry)); + } catch (InvalidShapeException e) { + // do not include geometry + } + // find dimensional-shape-type of geometry + CentroidCalculator centroidCalculator = new CentroidCalculator(geometry); + DimensionalShapeType geometryShapeType = centroidCalculator.getDimensionalShapeType(); + targetShapeType = targetShapeType.compareTo(geometryShapeType) >= 0 ? targetShapeType : geometryShapeType; + } + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + CompensatedSum compensatedSumLon = new CompensatedSum(0, 0); + CompensatedSum compensatedSumLat = new CompensatedSum(0, 0); + CompensatedSum compensatedSumWeight = new CompensatedSum(0, 0); + for (Geometry geometry : geometries) { + Document document = new Document(); + CentroidCalculator calculator = new CentroidCalculator(geometry); + document.add(new BinaryGeoShapeDocValuesField("field", GeoTestUtils.toDecodedTriangles(geometry), calculator)); + w.addDocument(document); + if (targetShapeType.compareTo(calculator.getDimensionalShapeType()) == 0) { + double weight = calculator.sumWeight(); + compensatedSumLat.add(weight * calculator.getY()); + compensatedSumLon.add(weight * calculator.getX()); + compensatedSumWeight.add(weight); + } + } + GeoPoint expectedCentroid = new GeoPoint(compensatedSumLat.value() / compensatedSumWeight.value(), + compensatedSumLon.value() / compensatedSumWeight.value()); + assertCentroid(w, expectedCentroid); + } + } + + private void assertCentroid(RandomIndexWriter w, GeoPoint expectedCentroid) throws IOException { + MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType(); + fieldType.setHasDocValues(true); + fieldType.setName("field"); + GeoCentroidAggregationBuilder aggBuilder = new GeoCentroidAggregationBuilder("my_agg") + .field("field"); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalGeoCentroid result = search(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType); + + assertEquals("my_agg", result.getName()); + GeoPoint centroid = result.centroid(); + assertNotNull(centroid); + assertEquals(expectedCentroid.getLat(), centroid.getLat(), GEOHASH_TOLERANCE); + assertEquals(expectedCentroid.getLon(), centroid.getLon(), GEOHASH_TOLERANCE); + assertTrue(AggregationInspectionHelper.hasValue(result)); + } + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new GeoCentroidAggregationBuilder("foo").field(fieldName); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(CoreValuesSourceType.GEOPOINT, GeoShapeValuesSourceType.instance()); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java index 0ce238a20be60..45fbb5fd944d1 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/CartesianFieldMapperTests.java @@ -21,7 +21,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.spatial.SpatialPlugin; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import java.io.IOException; import java.util.Collection; @@ -37,7 +37,7 @@ public abstract class CartesianFieldMapperTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class, SpatialPlugin.class, LocalStateCompositeXPackPlugin.class); + return pluginList(InternalSettingsPlugin.class, LocalStateSpatialPlugin.class, LocalStateCompositeXPackPlugin.class); } protected abstract XContentBuilder createDefaultMapping(String fieldName, diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java index 48d923b1a0261..b04681c079bdd 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapperTests.java @@ -42,7 +42,7 @@ import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.VersionUtils; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.spatial.SpatialPlugin; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import java.io.IOException; import java.util.Collection; @@ -62,7 +62,7 @@ protected boolean forbidPrivateIndexSettings() { @Override protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class, SpatialPlugin.class, LocalStateCompositeXPackPlugin.class); + return pluginList(InternalSettingsPlugin.class, LocalStateCompositeXPackPlugin.class, LocalStateSpatialPlugin.class); } public void testDefaultConfiguration() throws IOException { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index 7a30c99698642..c9a44b2ff5ecd 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -17,13 +17,8 @@ import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.spatial.SpatialPlugin; import java.io.IOException; -import java.util.Collection; import java.util.Collections; import static org.elasticsearch.index.mapper.GeoPointFieldMapper.Names.IGNORE_Z_VALUE; @@ -32,10 +27,6 @@ /** testing for {@link org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper} */ public class ShapeFieldMapperTests extends CartesianFieldMapperTests { - @Override - protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class, SpatialPlugin.class, LocalStateCompositeXPackPlugin.class); - } @Override protected XContentBuilder createDefaultMapping(String fieldName, diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderTests.java index bb55793ec85fc..45839ce933be7 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/query/ShapeQueryBuilderTests.java @@ -31,7 +31,7 @@ import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.AbstractQueryTestCase; -import org.elasticsearch.xpack.spatial.SpatialPlugin; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import org.junit.After; import java.io.IOException; @@ -62,7 +62,7 @@ public abstract class ShapeQueryBuilderTests extends AbstractQueryTestCase> getPlugins() { - return Arrays.asList(SpatialPlugin.class); + return Arrays.asList(LocalStateSpatialPlugin.class); } protected String fieldName() { diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java index 0cfe66eab4ec9..b3ef8de148c33 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryOverShapeTests.java @@ -22,15 +22,11 @@ import org.elasticsearch.geometry.ShapeType; import org.elasticsearch.index.query.ExistsQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.spatial.SpatialPlugin; import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder; import org.elasticsearch.xpack.spatial.util.ShapeTestUtils; import org.locationtech.jts.geom.Coordinate; import java.io.IOException; -import java.util.Collection; import java.util.Locale; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -196,11 +192,6 @@ public void testShapeFetchingPath() throws Exception { assertHitCount(result, 1); } - @Override - protected Collection> getPlugins() { - return pluginList(SpatialPlugin.class, LocalStateCompositeXPackPlugin.class); - } - /** * Test that ignore_malformed on GeoShapeFieldMapper does not fail the entire document */ diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTests.java index 1accbb68bba12..43d550282ac43 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/ShapeQueryTests.java @@ -27,7 +27,7 @@ import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.spatial.SpatialPlugin; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder; import org.locationtech.jts.geom.Coordinate; @@ -45,7 +45,7 @@ public abstract class ShapeQueryTests extends ESSingleNodeTestCase { @Override protected Collection> getPlugins() { - return pluginList(SpatialPlugin.class, LocalStateCompositeXPackPlugin.class); + return pluginList(LocalStateSpatialPlugin.class, LocalStateCompositeXPackPlugin.class); } protected abstract XContentBuilder createDefaultMapping() throws Exception; diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java index 439166c5250fd..218974f8c6a2c 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/metrics/GeoShapeBoundsAggregatorTests.java @@ -26,8 +26,9 @@ import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; import org.elasticsearch.search.aggregations.support.AggregationInspectionHelper; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.xpack.spatial.SpatialPlugin; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; import org.elasticsearch.xpack.spatial.index.fielddata.CentroidCalculator; import org.elasticsearch.xpack.spatial.index.mapper.BinaryGeoShapeDocValuesField; import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; @@ -46,7 +47,7 @@ public class GeoShapeBoundsAggregatorTests extends AggregatorTestCase { @Override protected List getSearchPlugins() { - return List.of(new SpatialPlugin()); + return List.of(new LocalStateSpatialPlugin()); } public void testEmpty() throws Exception { @@ -226,6 +227,6 @@ protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldTy @Override protected List getSupportedValuesSourceTypes() { - return List.of(GeoShapeValuesSourceType.instance()); + return List.of(CoreValuesSourceType.GEOPOINT, GeoShapeValuesSourceType.instance()); } } diff --git a/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/20_geo_centroid.yml b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/20_geo_centroid.yml new file mode 100644 index 0000000000000..a792ee33ff371 --- /dev/null +++ b/x-pack/plugin/spatial/src/test/resources/rest-api-spec/test/20_geo_centroid.yml @@ -0,0 +1,54 @@ +--- +"Test geo_centroid aggregation on geo_shape field": + - do: + indices.create: + index: locations + body: + mappings: + properties: + location: + type: geo_shape + + - do: + bulk: + refresh: true + body: + - index: + _index: locations + _id: 1 + - '{"location": "POINT(4.912350 52.374081)", "city": "Amsterdam", "name": "NEMO Science Museum"}' + - index: + _index: locations + _id: 2 + - '{"location": "POINT(4.901618 52.369219)", "city": "Amsterdam", "name": "Museum Het Rembrandthuis"}' + - index: + _index: locations + _id: 3 + - '{"location": "POINT(4.914722 52.371667)", "city": "Amsterdam", "name": "Nederlands Scheepvaartmuseum"}' + - index: + _index: locations + _id: 4 + - '{"location": "POINT(4.405200 51.222900)", "city": "Antwerp", "name": "Letterenhuis"}' + - index: + _index: locations + _id: 5 + - '{"location": "POINT(2.336389 48.861111)", "city": "Paris", "name": "Musée du Louvre"}' + - index: + _index: locations + _id: 6 + - '{"location": "POINT(2.327000 48.860000)", "city": "Paris", "name": "Musée dOrsay"}' + + - do: + search: + rest_total_hits_as_int: true + index: locations + size: 0 + body: + aggs: + centroid: + geo_centroid: + field: location + - match: {hits.total: 6 } + - match: { aggregations.centroid.location.lat: 51.00982965203002 } + - match: { aggregations.centroid.location.lon: 3.9662131341174245 } + - match: { aggregations.centroid.count: 6 }