Skip to content

Commit

Permalink
add doc_values mapping option to geo_shape field mapping (#47519)
Browse files Browse the repository at this point in the history 
This PR adds support for the `doc_values` field mapping parameter.

`true` and `false` supported by the GeoShapeFieldMapper,
only `false` is supported by the LegacyGeoShapeFieldMapper.

relates #37206
  • Loading branch information
@talevy
talevy committed Feb 24, 2020
1 parent 3d1e21b commit 77f9bb6
Show file tree
Hide file tree
Showing 8 changed files with 254 additions and 21 deletions.
5 changes: 5 additions & 0 deletions docs/reference/mapping/types/geo-shape.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,11 @@ and reject the whole document.
|`coerce` |If `true` unclosed linear rings in polygons will be automatically closed.
| `false`

|`doc_values` |Should the field be stored on disk in a column-stride fashion, so that it
can later be used for sorting, aggregations, or scripting? Accepts `true`
(default) or `false`.
| `true` for BKD-backed geo_shape, `false` for prefix tree indexing strategy

|=======================================================================


Expand Down
40 changes: 30 additions & 10 deletions server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public static class Defaults {
public static final Explicit<Boolean> COERCE = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_Z_VALUE = new Explicit<>(true, false);
public static final Explicit<Boolean> DOC_VALUES = new Explicit<>(false, false);
}


Expand Down Expand Up @@ -122,15 +123,6 @@ public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFi
super(name, fieldType, defaultFieldType);
}

public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType,
boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ) {
super(name, fieldType, defaultFieldType);
this.coerce = coerce;
this.ignoreMalformed = ignoreMalformed;
this.orientation = orientation;
this.ignoreZValue = ignoreZ;
}

public Builder coerce(boolean coerce) {
this.coerce = coerce;
return this;
Expand Down Expand Up @@ -190,6 +182,15 @@ public Builder ignoreZValue(final boolean ignoreZValue) {
return this;
}

protected Explicit<Boolean> docValues() {
if (docValuesSet && fieldType.hasDocValues()) {
return new Explicit<>(true, true);
} else if (docValuesSet) {
return new Explicit<>(false, true);
}
return Defaults.DOC_VALUES;
}

@Override
protected void setupFieldType(BuilderContext context) {
super.setupFieldType(context);
Expand Down Expand Up @@ -251,13 +252,20 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
XContentMapValues.nodeBooleanValue(fieldNode,
name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName()));
iterator.remove();
} else if (TypeParsers.DOC_VALUES.equals(fieldName)) {
params.put(TypeParsers.DOC_VALUES, XContentMapValues.nodeBooleanValue(fieldNode, name + "." + TypeParsers.DOC_VALUES));
iterator.remove();
}
}
if (parserContext.indexVersionCreated().onOrAfter(Version.V_6_6_0) && parsedDeprecatedParameters == false) {
params.remove(DEPRECATED_PARAMETERS_KEY);
}
Builder builder = newBuilder(name, params);

if (params.containsKey(TypeParsers.DOC_VALUES)) {
builder.docValues((Boolean) params.get(TypeParsers.DOC_VALUES));
}

if (params.containsKey(Names.COERCE.getPreferredName())) {
builder.coerce((Boolean)params.get(Names.COERCE.getPreferredName()));
}
Expand Down Expand Up @@ -358,15 +366,17 @@ public QueryProcessor geometryQueryBuilder() {
protected Explicit<Boolean> coerce;
protected Explicit<Boolean> ignoreMalformed;
protected Explicit<Boolean> ignoreZValue;
protected Explicit<Boolean> docValues;

protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
this.coerce = coerce;
this.ignoreMalformed = ignoreMalformed;
this.ignoreZValue = ignoreZValue;
this.docValues = docValues;
}

@Override
Expand All @@ -382,6 +392,9 @@ protected void doMerge(Mapper mergeWith) {
if (gsfm.ignoreZValue.explicit()) {
this.ignoreZValue = gsfm.ignoreZValue;
}
if (gsfm.docValues.explicit()) {
this.docValues = gsfm.docValues;
}
}

@Override
Expand All @@ -405,6 +418,9 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par
if (includeDefaults || ignoreZValue.explicit()) {
builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value());
}
if (includeDefaults || docValues.explicit()) {
builder.field(TypeParsers.DOC_VALUES, docValues.value());
}
}

public Explicit<Boolean> coerce() {
Expand All @@ -419,6 +435,10 @@ public Explicit<Boolean> ignoreZValue() {
return ignoreZValue;
}

public Explicit<Boolean> docValues() {
return docValues;
}

public Orientation orientation() {
return ((AbstractGeometryFieldType)fieldType).orientation();
}
Expand Down
17 changes: 14 additions & 3 deletions server/src/main/java/org/elasticsearch/index/mapper/GeoShapeFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,25 @@ public Builder(String name) {
public GeoShapeFieldMapper build(BuilderContext context) {
setupFieldType(context);
return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context),
ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
ignoreZValue(), docValues(), context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
}

@Override
public boolean defaultDocValues(Version indexCreated) {
return Version.CURRENT.onOrBefore(indexCreated);
}

@Override
protected Explicit<Boolean> docValues() {
if (docValuesSet && fieldType.hasDocValues()) {
return new Explicit<>(true, true);
} else if (docValuesSet) {
return new Explicit<>(false, true);
}
return new Explicit<>(fieldType.hasDocValues(), false);
}

@Override
protected void setupFieldType(BuilderContext context) {
super.setupFieldType(context);
Expand Down Expand Up @@ -132,9 +143,9 @@ public String typeName() {

public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings,
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, indexSettings,
multiFields, copyTo);
}

Expand Down
19 changes: 15 additions & 4 deletions server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,16 @@ public GeoShapeFieldType fieldType() {
return (GeoShapeFieldType)fieldType;
}

public Builder docValues(boolean hasDocValues) {
super.docValues(hasDocValues);
if (hasDocValues) {
throw new ElasticsearchParseException("geo_shape field [" + name
+ "] indexed using prefix-trees do not support doc_values");
}
// doc-values already set to `false`
return this;
}

private void setupFieldTypeDeprecatedParameters(BuilderContext context) {
GeoShapeFieldType ft = fieldType();
if (deprecatedParameters.strategy != null) {
Expand Down Expand Up @@ -294,7 +304,7 @@ public LegacyGeoShapeFieldMapper build(BuilderContext context) {
setupFieldType(context);

return new LegacyGeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context),
coerce(context), orientation(), ignoreZValue(), context.indexSettings(),
coerce(context), orientation(), ignoreZValue(), docValues(), context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
}
}
Expand All @@ -320,6 +330,7 @@ public GeoShapeFieldType() {
setStored(false);
setStoreTermVectors(false);
setOmitNorms(true);
setHasDocValues(false);
}

protected GeoShapeFieldType(GeoShapeFieldType ref) {
Expand Down Expand Up @@ -472,10 +483,10 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) {

public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, Explicit<Orientation> orientation,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings,
multiFields, copyTo);
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues,
indexSettings, multiFields, copyTo);
}

@Override
Expand Down
60 changes: 59 additions & 1 deletion server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ public void testDefaultConfiguration() throws IOException {
GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
assertThat(geoShapeFieldMapper.fieldType().orientation(),
equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value()));
assertTrue(geoShapeFieldMapper.fieldType.hasDocValues());
assertFalse(geoShapeFieldMapper.docValues().explicit());
assertTrue(geoShapeFieldMapper.docValues().value());
assertTrue(geoShapeFieldMapper.fieldType().hasDocValues());
}

/**
Expand Down Expand Up @@ -214,6 +216,45 @@ public void testIgnoreMalformedParsing() throws IOException {
assertThat(ignoreMalformed.value(), equalTo(false));
}

/**
* Test that doc_values parameter correctly parses
*/
public void testDocValues() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("doc_values", true)
.endObject().endObject()
.endObject().endObject());

DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
.parse("type1", new CompressedXContent(mapping));
Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));

assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().explicit());
assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().value());
boolean hasDocValues = ((GeoShapeFieldMapper)fieldMapper).fieldType().hasDocValues();
assertTrue(hasDocValues);

// explicit false doc_values
mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("doc_values", "false")
.endObject().endObject()
.endObject().endObject());

defaultMapper = createIndex("test2").mapperService().documentMapperParser()
.parse("type1", new CompressedXContent(mapping));
fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));

assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().explicit());
assertFalse(((GeoShapeFieldMapper)fieldMapper).docValues().value());
hasDocValues = ((GeoShapeFieldMapper)fieldMapper).fieldType().hasDocValues();
assertFalse(hasDocValues);
}

private void assertFieldWarnings(String... fieldNames) {
String[] warnings = new String[fieldNames.length];
Expand Down Expand Up @@ -283,9 +324,26 @@ public void testSerializeDefaults() throws Exception {
String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location"));
assertTrue(serialized, serialized.contains("\"orientation\":\"" +
AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\""));
assertTrue(serialized, serialized.contains("\"doc_values\":true"));
}
}

public void testSerializeDocValues() throws IOException {
boolean docValues = randomBoolean();
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("doc_values", docValues)
.endObject().endObject()
.endObject().endObject());
DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping));
String serialized = toXContentString((GeoShapeFieldMapper) mapper.mappers().getMapper("location"));
assertTrue(serialized, serialized.contains("\"orientation\":\"" +
AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\""));
assertTrue(serialized, serialized.contains("\"doc_values\":" + docValues));
}

public String toXContentString(GeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
ToXContent.Params params;
Expand Down
Loading

0 comments on commit 77f9bb6

Please sign in to comment.