Skip to content

Commit

Permalink
[ML] Transition to typeless (mapping) APIs (#39572)
Browse files Browse the repository at this point in the history 
ML has historically used doc as the single mapping type but reindex in 7.x
will change the mapping to _doc. Switching to the typeless APIs handles 
case where the mapping type is either doc or _doc. This change removes
deprecated typed usages.
  • Loading branch information
@davidkyle
davidkyle authored Mar 4, 2019
1 parent 7dfca10 commit d6b55ef
Show file tree
Hide file tree
Showing 52 changed files with 216 additions and 362 deletions.
5 changes: 2 additions & 3 deletions x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.io.IOException;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;

public final class MlMetaIndex {
/**
Expand All @@ -21,14 +22,12 @@ public final class MlMetaIndex {
*/
public static final String INDEX_NAME = ".ml-meta";

public static final String TYPE = "doc";

private MlMetaIndex() {}

public static XContentBuilder docMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
ElasticsearchMappings.addDefaultMapping(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
Expand Down
5 changes: 3 additions & 2 deletions ...lugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.SortedMap;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;

Expand Down Expand Up @@ -71,7 +72,7 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c

CreateIndexRequest createIndexRequest = new CreateIndexRequest(INDEX_NAME);
try (XContentBuilder annotationsMapping = AnnotationIndex.annotationsMapping()) {
createIndexRequest.mapping(ElasticsearchMappings.DOC_TYPE, annotationsMapping);
createIndexRequest.mapping(SINGLE_MAPPING_NAME, annotationsMapping);
createIndexRequest.settings(Settings.builder()
.put(IndexMetaData.SETTING_AUTO_EXPAND_REPLICAS, "0-1")
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, "1")
Expand Down Expand Up @@ -111,7 +112,7 @@ public static void createAnnotationsIndexIfNecessary(Settings settings, Client c
public static XContentBuilder annotationsMapping() throws IOException {
XContentBuilder builder = jsonBuilder()
.startObject()
.startObject(ElasticsearchMappings.DOC_TYPE);
.startObject(SINGLE_MAPPING_NAME);
ElasticsearchMappings.addMetaInformation(builder);
builder.startObject(ElasticsearchMappings.PROPERTIES)
.startObject(Annotation.ANNOTATION.getPreferredName())
Expand Down
105 changes: 59 additions & 46 deletions .../src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import org.elasticsearch.cluster.metadata.AliasOrIndex;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.CheckedSupplier;
import org.elasticsearch.common.CheckedBiFunction;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.Index;
Expand Down Expand Up @@ -62,6 +62,7 @@
import java.util.Map;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN;
import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin;

Expand All @@ -86,8 +87,6 @@
*/
public class ElasticsearchMappings {

public static final String DOC_TYPE = "doc";

/**
* String constants used in mappings
*/
Expand Down Expand Up @@ -137,7 +136,7 @@ private ElasticsearchMappings() {
public static XContentBuilder configMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
Expand Down Expand Up @@ -420,14 +419,14 @@ public static void addMetaInformation(XContentBuilder builder) throws IOExceptio
.endObject();
}

public static XContentBuilder resultsMapping() throws IOException {
return resultsMapping(Collections.emptyList());
public static XContentBuilder resultsMapping(String mappingType) throws IOException {
return resultsMapping(mappingType, Collections.emptyList());
}

public static XContentBuilder resultsMapping(Collection<String> extraTermFields) throws IOException {
public static XContentBuilder resultsMapping(String mappingType, Collection<String> extraTermFields) throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(mappingType);
addMetaInformation(builder);
addDefaultMapping(builder);
builder.startObject(PROPERTIES);
Expand Down Expand Up @@ -456,11 +455,12 @@ public static XContentBuilder resultsMapping(Collection<String> extraTermFields)

// end properties
builder.endObject();
// end mapping
// end type
builder.endObject();
// end doc
// end mapping
builder.endObject();


return builder;
}

Expand Down Expand Up @@ -575,18 +575,25 @@ private static void addResultsMapping(XContentBuilder builder) throws IOExceptio
addModelSizeStatsFieldsToMapping(builder);
}

public static XContentBuilder termFieldsMapping(String type, Collection<String> termFields) {
/**
* Generate a keyword mapping for {@code termFields} for the default type
* {@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME}
*
* If the returned mapping is used in index creation and the new index has a matching template
* then the mapping type ({@link org.elasticsearch.index.mapper.MapperService#SINGLE_MAPPING_NAME})
* must match the mapping type of the template otherwise the mappings will not be merged correctly.
*
* @param termFields Fields to generate mapping for
* @return The mapping
*/
public static XContentBuilder termFieldsMapping(Collection<String> termFields) {
try {
XContentBuilder builder = jsonBuilder().startObject();
if (type != null) {
builder.startObject(type);
}
builder.startObject(SINGLE_MAPPING_NAME);
builder.startObject(PROPERTIES);
addTermFields(builder, termFields);
builder.endObject();
if (type != null) {
builder.endObject();
}
builder.endObject();
return builder.endObject();
} catch (IOException e) {
throw new RuntimeException(e);
Expand Down Expand Up @@ -872,7 +879,7 @@ private static void addCategoryDefinitionMapping(XContentBuilder builder) throws
public static XContentBuilder stateMapping() throws IOException {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject(DOC_TYPE);
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
builder.field(ENABLED, false);
builder.endObject();
Expand Down Expand Up @@ -960,46 +967,47 @@ private static void addModelSizeStatsFieldsToMapping(XContentBuilder builder) th
}

public static XContentBuilder auditMessageMapping() throws IOException {
XContentBuilder builder = jsonBuilder().startObject()
.startObject(AuditMessage.TYPE.getPreferredName());
XContentBuilder builder = jsonBuilder().startObject();
builder.startObject(SINGLE_MAPPING_NAME);
addMetaInformation(builder);
builder.startObject(PROPERTIES)
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.LEVEL.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.MESSAGE.getPreferredName())
.field(TYPE, TEXT)
.startObject(FIELDS)
.startObject(RAW)
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject()
.startObject(AuditMessage.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(AuditMessage.NODE_NAME.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(Job.ID.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.LEVEL.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.startObject(AuditMessage.MESSAGE.getPreferredName())
.field(TYPE, TEXT)
.startObject(FIELDS)
.startObject(RAW)
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject();
.endObject()
.startObject(AuditMessage.TIMESTAMP.getPreferredName())
.field(TYPE, DATE)
.endObject()
.startObject(AuditMessage.NODE_NAME.getPreferredName())
.field(TYPE, KEYWORD)
.endObject()
.endObject()
.endObject()
.endObject();

return builder;
}

static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion) throws IOException {
List<String> indicesToUpdate = new ArrayList<>();

ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> currentMapping = state.metaData().findMappings(concreteIndices,
new String[] {DOC_TYPE}, MapperPlugin.NOOP_FIELD_FILTER);
new String[0], MapperPlugin.NOOP_FIELD_FILTER);

for (String index : concreteIndices) {
ImmutableOpenMap<String, MappingMetaData> innerMap = currentMapping.get(index);
if (innerMap != null) {
MappingMetaData metaData = innerMap.get(DOC_TYPE);
MappingMetaData metaData = innerMap.valuesIt().next();
try {
@SuppressWarnings("unchecked")
Map<String, Object> meta = (Map<String, Object>) metaData.sourceAsMap().get("_meta");
Expand Down Expand Up @@ -1038,7 +1046,8 @@ static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndic
return indicesToUpdate.toArray(new String[indicesToUpdate.size()]);
}

public static void addDocMappingIfMissing(String alias, CheckedSupplier<XContentBuilder, IOException> mappingSupplier,
public static void addDocMappingIfMissing(String alias,
CheckedBiFunction<String, Collection<String>, XContentBuilder, IOException> mappingSupplier,
Client client, ClusterState state, ActionListener<Boolean> listener) {
AliasOrIndex aliasOrIndex = state.metaData().getAliasAndIndexLookup().get(alias);
if (aliasOrIndex == null) {
Expand All @@ -1058,9 +1067,13 @@ public static void addDocMappingIfMissing(String alias, CheckedSupplier<XContent
}

if (indicesThatRequireAnUpdate.length > 0) {
try (XContentBuilder mapping = mappingSupplier.get()) {
// Use the mapping type of the first index in the update
IndexMetaData indexMetaData = state.metaData().index(indicesThatRequireAnUpdate[0]);
String mappingType = indexMetaData.mapping().type();

try (XContentBuilder mapping = mappingSupplier.apply(mappingType, Collections.emptyList())) {
PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate);
putMappingRequest.type(DOC_TYPE);
putMappingRequest.type(mappingType);
putMappingRequest.source(mapping);
executeAsyncWithOrigin(client, ML_ORIGIN, PutMappingAction.INSTANCE, putMappingRequest,
ActionListener.wrap(response -> {
Expand Down
2 changes: 1 addition & 1 deletion ...plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/AuditMessage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import java.util.Objects;

public class AuditMessage implements ToXContentObject, Writeable {
public static final ParseField TYPE = new ParseField("audit_message");
private static final ParseField TYPE = new ParseField("audit_message");

public static final ParseField MESSAGE = new ParseField("message");
public static final ParseField LEVEL = new ParseField("level");
Expand Down
11 changes: 7 additions & 4 deletions ...test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
import java.util.Map;
import java.util.Set;

import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;


public class ElasticsearchMappingsTests extends ESTestCase {

Expand Down Expand Up @@ -117,11 +119,12 @@ private void compareFields(Set<String> expected, Set<String> reserved) {
@SuppressWarnings("unchecked")
public void testTermFieldMapping() throws IOException {

XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(null, Arrays.asList("apple", "strawberry",
XContentBuilder builder = ElasticsearchMappings.termFieldsMapping(Arrays.asList("apple", "strawberry",
AnomalyRecord.BUCKET_SPAN.getPreferredName()));

XContentParser parser = createParser(builder);
Map<String, Object> properties = (Map<String, Object>) parser.map().get(ElasticsearchMappings.PROPERTIES);
Map<String, Object> mapping = (Map<String, Object>) parser.map().get(SINGLE_MAPPING_NAME);
Map<String, Object> properties = (Map<String, Object>) mapping.get(ElasticsearchMappings.PROPERTIES);

Map<String, Object> instanceMapping = (Map<String, Object>) properties.get("apple");
assertNotNull(instanceMapping);
Expand Down Expand Up @@ -217,7 +220,7 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map<String, Object>
}
mapping.put("_meta", meta);

indexMetaData.putMapping(new MappingMetaData(ElasticsearchMappings.DOC_TYPE, mapping));
indexMetaData.putMapping(new MappingMetaData("_doc", mapping));

metaDataBuilder.put(indexMetaData);
}
Expand All @@ -230,7 +233,7 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map<String, Object>

private Set<String> collectResultsDocFieldNames() throws IOException {
// Only the mappings for the results index should be added below. Do NOT add mappings for other indexes here.
return collectFieldNames(ElasticsearchMappings.resultsMapping());
return collectFieldNames(ElasticsearchMappings.resultsMapping("_doc"));
}

private Set<String> collectConfigDocFieldNames() throws IOException {
Expand Down
14 changes: 7 additions & 7 deletions ...lti-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.List;
import java.util.Locale;

import static org.elasticsearch.index.mapper.MapperService.SINGLE_MAPPING_NAME;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;

Expand All @@ -35,38 +36,37 @@
public class CategorizationIT extends MlNativeAutodetectIntegTestCase {

private static final String DATA_INDEX = "log-data";
private static final String DATA_TYPE = "log";

private long nowMillis;

@Before
public void setUpData() {
client().admin().indices().prepareCreate(DATA_INDEX)
.addMapping(DATA_TYPE, "time", "type=date,format=epoch_millis",
.addMapping(SINGLE_MAPPING_NAME, "time", "type=date,format=epoch_millis",
"msg", "type=text")
.get();

nowMillis = System.currentTimeMillis();

BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
IndexRequest indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
IndexRequest indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(2).millis(),
"msg", "Node 1 started");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(2).millis() + 1,
"msg", "Failed to shutdown [error org.aaaa.bbbb.Cccc line 54 caused " +
"by foo exception]");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(1).millis(),
"msg", "Node 2 started");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis - TimeValue.timeValueHours(1).millis() + 1,
"msg", "Failed to shutdown [error but this time completely different]");
bulkRequestBuilder.add(indexRequest);
indexRequest = new IndexRequest(DATA_INDEX, DATA_TYPE);
indexRequest = new IndexRequest(DATA_INDEX);
indexRequest.source("time", nowMillis, "msg", "Node 3 started");
bulkRequestBuilder.add(indexRequest);

Expand Down
Loading

0 comments on commit d6b55ef

Please sign in to comment.