Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add doc_values mapping option to geo_shape field mapping #47519

Merged
merged 11 commits into from
Feb 24, 2020
5 changes: 5 additions & 0 deletions docs/reference/mapping/types/geo-shape.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,11 @@ and reject the whole document.
|`coerce` |If `true` unclosed linear rings in polygons will be automatically closed.
| `false`

|`doc_values` |Should the field be stored on disk in a column-stride fashion, so that it
can later be used for sorting, aggregations, or scripting? Accepts `true`
(default) or `false`.
| `true` for BKD-backed geo_shape, `false` for prefix tree indexing strategy

|=======================================================================


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ public static class Defaults {
public static final Explicit<Boolean> COERCE = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_MALFORMED = new Explicit<>(false, false);
public static final Explicit<Boolean> IGNORE_Z_VALUE = new Explicit<>(true, false);
public static final Explicit<Boolean> DOC_VALUES = new Explicit<>(false, false);
}


Expand Down Expand Up @@ -122,15 +123,6 @@ public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFi
super(name, fieldType, defaultFieldType);
}

public Builder(String name, MappedFieldType fieldType, MappedFieldType defaultFieldType,
boolean coerce, boolean ignoreMalformed, Orientation orientation, boolean ignoreZ) {
super(name, fieldType, defaultFieldType);
this.coerce = coerce;
this.ignoreMalformed = ignoreMalformed;
this.orientation = orientation;
this.ignoreZValue = ignoreZ;
}

public Builder coerce(boolean coerce) {
this.coerce = coerce;
return this;
Expand Down Expand Up @@ -190,6 +182,15 @@ public Builder ignoreZValue(final boolean ignoreZValue) {
return this;
}

protected Explicit<Boolean> docValues() {
if (docValuesSet && fieldType.hasDocValues()) {
return new Explicit<>(true, true);
} else if (docValuesSet) {
return new Explicit<>(false, true);
}
return Defaults.DOC_VALUES;
}

@Override
protected void setupFieldType(BuilderContext context) {
super.setupFieldType(context);
Expand Down Expand Up @@ -251,13 +252,20 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
XContentMapValues.nodeBooleanValue(fieldNode,
name + "." + GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName()));
iterator.remove();
} else if (TypeParsers.DOC_VALUES.equals(fieldName)) {
params.put(TypeParsers.DOC_VALUES, XContentMapValues.nodeBooleanValue(fieldNode, name + "." + TypeParsers.DOC_VALUES));
iterator.remove();
}
}
if (parsedDeprecatedParameters == false) {
params.remove(DEPRECATED_PARAMETERS_KEY);
}
Builder builder = newBuilder(name, params);

if (params.containsKey(TypeParsers.DOC_VALUES)) {
builder.docValues((Boolean) params.get(TypeParsers.DOC_VALUES));
}

if (params.containsKey(Names.COERCE.getPreferredName())) {
builder.coerce((Boolean)params.get(Names.COERCE.getPreferredName()));
}
Expand Down Expand Up @@ -358,15 +366,17 @@ public QueryProcessor geometryQueryBuilder() {
protected Explicit<Boolean> coerce;
protected Explicit<Boolean> ignoreMalformed;
protected Explicit<Boolean> ignoreZValue;
protected Explicit<Boolean> docValues;

protected AbstractGeometryFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo);
this.coerce = coerce;
this.ignoreMalformed = ignoreMalformed;
this.ignoreZValue = ignoreZValue;
this.docValues = docValues;
}

@Override
Expand All @@ -382,6 +392,9 @@ protected void doMerge(Mapper mergeWith) {
if (gsfm.ignoreZValue.explicit()) {
this.ignoreZValue = gsfm.ignoreZValue;
}
if (gsfm.docValues.explicit()) {
this.docValues = gsfm.docValues;
}
}

@Override
Expand All @@ -405,6 +418,9 @@ public void doXContentBody(XContentBuilder builder, boolean includeDefaults, Par
if (includeDefaults || ignoreZValue.explicit()) {
builder.field(GeoPointFieldMapper.Names.IGNORE_Z_VALUE.getPreferredName(), ignoreZValue.value());
}
if (includeDefaults || docValues.explicit()) {
builder.field(TypeParsers.DOC_VALUES, docValues.value());
}
}

public Explicit<Boolean> coerce() {
Expand All @@ -419,6 +435,10 @@ public Explicit<Boolean> ignoreZValue() {
return ignoreZValue;
}

public Explicit<Boolean> docValues() {
return docValues;
}

public Orientation orientation() {
return ((AbstractGeometryFieldType)fieldType).orientation();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,14 +67,24 @@ public Builder(String name) {
public GeoShapeFieldMapper build(BuilderContext context) {
setupFieldType(context);
return new GeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context), coerce(context),
ignoreZValue(), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
ignoreZValue(), docValues(), context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
}

@Override
public boolean defaultDocValues(Version indexCreated) {
return Version.V_8_0_0.onOrBefore(indexCreated);
}

protected Explicit<Boolean> docValues() {
if (docValuesSet && fieldType.hasDocValues()) {
return new Explicit<>(true, true);
} else if (docValuesSet) {
return new Explicit<>(false, true);
}
return new Explicit<>(fieldType.hasDocValues(), false);
}

protected void setupFieldType(BuilderContext context) {
super.setupFieldType(context);

Expand Down Expand Up @@ -131,9 +141,9 @@ public String typeName() {

public GeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings,
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues, indexSettings,
multiFields, copyTo);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,16 @@ public GeoShapeFieldType fieldType() {
return (GeoShapeFieldType)fieldType;
}

public Builder docValues(boolean hasDocValues) {
super.docValues(hasDocValues);
if (hasDocValues) {
throw new ElasticsearchParseException("geo_shape field [" + name
+ "] indexed using prefix-trees do not support doc_values");
}
// doc-values already set to `false`
return this;
}

private void setupFieldTypeDeprecatedParameters(BuilderContext context) {
GeoShapeFieldType ft = fieldType();
if (deprecatedParameters.strategy != null) {
Expand Down Expand Up @@ -292,7 +302,7 @@ public LegacyGeoShapeFieldMapper build(BuilderContext context) {
setupFieldType(context);

return new LegacyGeoShapeFieldMapper(name, fieldType, defaultFieldType, ignoreMalformed(context),
coerce(context), orientation(), ignoreZValue(), context.indexSettings(),
coerce(context), orientation(), ignoreZValue(), docValues(), context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
}
}
Expand All @@ -318,6 +328,7 @@ public GeoShapeFieldType() {
setStored(false);
setStoreTermVectors(false);
setOmitNorms(true);
setHasDocValues(false);
}

protected GeoShapeFieldType(GeoShapeFieldType ref) {
Expand Down Expand Up @@ -470,10 +481,10 @@ public PrefixTreeStrategy resolvePrefixTreeStrategy(String strategyName) {

public LegacyGeoShapeFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, Explicit<Orientation> orientation,
Explicit<Boolean> ignoreZValue, Settings indexSettings,
Explicit<Boolean> ignoreZValue, Explicit<Boolean> docValues, Settings indexSettings,
MultiFields multiFields, CopyTo copyTo) {
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, indexSettings,
multiFields, copyTo);
super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, ignoreZValue, docValues,
indexSettings, multiFields, copyTo);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,9 @@ public void testDefaultConfiguration() throws IOException {
GeoShapeFieldMapper geoShapeFieldMapper = (GeoShapeFieldMapper) fieldMapper;
assertThat(geoShapeFieldMapper.fieldType().orientation(),
equalTo(GeoShapeFieldMapper.Defaults.ORIENTATION.value()));
assertTrue(geoShapeFieldMapper.fieldType.hasDocValues());
assertFalse(geoShapeFieldMapper.docValues().explicit());
assertTrue(geoShapeFieldMapper.docValues().value());
assertTrue(geoShapeFieldMapper.fieldType().hasDocValues());
}

/**
Expand Down Expand Up @@ -214,6 +216,45 @@ public void testIgnoreMalformedParsing() throws IOException {
assertThat(ignoreMalformed.value(), equalTo(false));
}

/**
* Test that doc_values parameter correctly parses
*/
public void testDocValues() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("doc_values", true)
.endObject().endObject()
.endObject().endObject());

DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser()
.parse("type1", new CompressedXContent(mapping));
Mapper fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));

assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().explicit());
assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().value());
boolean hasDocValues = ((GeoShapeFieldMapper)fieldMapper).fieldType().hasDocValues();
assertTrue(hasDocValues);

// explicit false doc_values
mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("doc_values", "false")
.endObject().endObject()
.endObject().endObject());

defaultMapper = createIndex("test2").mapperService().documentMapperParser()
.parse("type1", new CompressedXContent(mapping));
fieldMapper = defaultMapper.mappers().getMapper("location");
assertThat(fieldMapper, instanceOf(GeoShapeFieldMapper.class));

assertTrue(((GeoShapeFieldMapper)fieldMapper).docValues().explicit());
assertFalse(((GeoShapeFieldMapper)fieldMapper).docValues().value());
hasDocValues = ((GeoShapeFieldMapper)fieldMapper).fieldType().hasDocValues();
assertFalse(hasDocValues);
}

private void assertFieldWarnings(String... fieldNames) {
String[] warnings = new String[fieldNames.length];
Expand Down Expand Up @@ -283,9 +324,26 @@ public void testSerializeDefaults() throws Exception {
String serialized = toXContentString((GeoShapeFieldMapper) defaultMapper.mappers().getMapper("location"));
assertTrue(serialized, serialized.contains("\"orientation\":\"" +
AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\""));
assertTrue(serialized, serialized.contains("\"doc_values\":true"));
}
}

public void testSerializeDocValues() throws IOException {
boolean docValues = randomBoolean();
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("type1")
.startObject("properties").startObject("location")
.field("type", "geo_shape")
.field("doc_values", docValues)
.endObject().endObject()
.endObject().endObject());
DocumentMapper mapper = parser.parse("type1", new CompressedXContent(mapping));
String serialized = toXContentString((GeoShapeFieldMapper) mapper.mappers().getMapper("location"));
assertTrue(serialized, serialized.contains("\"orientation\":\"" +
AbstractGeometryFieldMapper.Defaults.ORIENTATION.value() + "\""));
assertTrue(serialized, serialized.contains("\"doc_values\":" + docValues));
}

public String toXContentString(GeoShapeFieldMapper mapper, boolean includeDefaults) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
ToXContent.Params params;
Expand Down
Loading