Skip to content

Commit

Permalink
Inject the host.name field mapping only if required for logsdb in…
Browse files Browse the repository at this point in the history 
…dex mode (elastic#114573)

Here we check for the existence of a `host.name` field in index sort settings
when the index mode is `logsdb` and decide to inject the field in the mapping
depending on whether it exists or not. By default `host.name` is required for
sorting in LogsDB. This reduces the chances for errors at mapping or template
composition time as a result of injecting the `host.name` field only if strictly
required. A user who wants to override index sort settings without including
a `host.name` field would be able to do so without finding an additional
`host.name` field in the mappings (injected automatically). If users override the
sort settings and a `host.name` field is not included we don't need
to inject such field since sorting does not require it anymore.

As a result of this change we have the following:
* the user does not provide any index sorting configuration: we are responsible for injecting the default sort fields and their mapping (for `logsdb`)
* the user explicitly provides non-empty index sorting configuration: the user is also responsible for providing correct mappings and we do not modify index sorting or mappings

Note also that all sort settings `index.sort.*` are `final` which means doing this
check once, when mappings are merged at template composition time, is enough.
  • Loading branch information
@salvatore-campagna
salvatore-campagna authored Oct 16, 2024
1 parent ff7ea10 commit 9bf6e3b
Show file tree
Hide file tree
Showing 6 changed files with 856 additions and 46 deletions.
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1373,7 +1373,7 @@ private static void updateIndexMappingsAndBuildSortOrder(
MapperService mapperService = indexService.mapperService();
IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD;
List<CompressedXContent> allMappings = new ArrayList<>();
final CompressedXContent defaultMapping = indexMode.getDefaultMapping();
final CompressedXContent defaultMapping = indexMode.getDefaultMapping(indexService.getIndexSettings());
if (defaultMapping != null) {
allMappings.add(defaultMapping);
}
Expand Down
71 changes: 30 additions & 41 deletions server/src/main/java/org/elasticsearch/index/IndexMode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
}

@Override
public CompressedXContent getDefaultMapping() {
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
return null;
}

Expand Down Expand Up @@ -171,7 +171,7 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
}

@Override
public CompressedXContent getDefaultMapping() {
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
}

Expand Down Expand Up @@ -249,8 +249,10 @@ public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup ma
}

@Override
public CompressedXContent getDefaultMapping() {
return DEFAULT_LOGS_TIMESTAMP_MAPPING;
public CompressedXContent getDefaultMapping(final IndexSettings indexSettings) {
return indexSettings != null && indexSettings.getIndexSortConfig().hasPrimarySortOnField(HOST_NAME)
? DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME
: DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
}

@Override
Expand Down Expand Up @@ -308,6 +310,8 @@ public String getDefaultCodec() {
}
};

private static final String HOST_NAME = "host.name";

private static void validateTimeSeriesSettings(Map<Setting<?>, Object> settings) {
settingRequiresTimeSeries(settings, IndexMetadata.INDEX_ROUTING_PATH);
settingRequiresTimeSeries(settings, IndexSettings.TIME_SERIES_START_TIME);
Expand All @@ -324,48 +328,33 @@ protected static String tsdbMode() {
return "[" + IndexSettings.MODE.getKey() + "=time_series]";
}

public static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;
private static CompressedXContent createDefaultMapping(boolean includeHostName) throws IOException {
return new CompressedXContent((builder, params) -> {
builder.startObject(MapperService.SINGLE_MAPPING_NAME)
.startObject(DataStreamTimestampFieldMapper.NAME)
.field("enabled", true)
.endObject()
.startObject("properties")
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
.field("type", DateFieldMapper.CONTENT_TYPE)
.endObject();

if (includeHostName) {
builder.startObject(HOST_NAME).field("type", KeywordFieldMapper.CONTENT_TYPE).field("ignore_above", 1024).endObject();
}

static {
try {
DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = new CompressedXContent(
((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
.startObject(DataStreamTimestampFieldMapper.NAME)
.field("enabled", true)
.endObject()
.startObject("properties")
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
.field("type", DateFieldMapper.CONTENT_TYPE)
.field("ignore_malformed", "false")
.endObject()
.endObject()
.endObject())
);
} catch (IOException e) {
throw new AssertionError(e);
}
return builder.endObject().endObject();
});
}

public static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING;
private static final CompressedXContent DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING;

private static final CompressedXContent DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME;

static {
try {
DEFAULT_LOGS_TIMESTAMP_MAPPING = new CompressedXContent(
((builder, params) -> builder.startObject(MapperService.SINGLE_MAPPING_NAME)
.startObject(DataStreamTimestampFieldMapper.NAME)
.field("enabled", true)
.endObject()
.startObject("properties")
.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH)
.field("type", DateFieldMapper.CONTENT_TYPE)
.endObject()
.startObject("host.name")
.field("type", KeywordFieldMapper.CONTENT_TYPE)
.field("ignore_above", 1024)
.endObject()
.endObject()
.endObject())
);
DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = createDefaultMapping(false);
DEFAULT_LOGS_TIMESTAMP_MAPPING_WITH_HOSTNAME = createDefaultMapping(true);
} catch (IOException e) {
throw new AssertionError(e);
}
Expand Down Expand Up @@ -421,7 +410,7 @@ public String getName() {
* Get default mapping for this index or {@code null} if there is none.
*/
@Nullable
public abstract CompressedXContent getDefaultMapping();
public abstract CompressedXContent getDefaultMapping(IndexSettings indexSettings);

/**
* Build the {@link FieldMapper} for {@code _id}.
Expand Down
16 changes: 14 additions & 2 deletions server/src/test/java/org/elasticsearch/index/LogsIndexModeTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,24 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.test.ESTestCase;

import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;

public class LogsIndexModeTests extends ESTestCase {
public void testLogsIndexModeSetting() {
assertThat(IndexSettings.MODE.get(buildSettings()), equalTo(IndexMode.LOGSDB));
}

public void testSortField() {
public void testDefaultHostNameSortField() {
final IndexMetadata metadata = IndexSettingsTests.newIndexMeta("test", buildSettings());
assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(true));
assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), containsString("host.name"));
}

public void testCustomSortField() {
final Settings sortSettings = Settings.builder()
.put(buildSettings())
.put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "agent_id")
Expand All @@ -29,7 +39,9 @@ public void testSortField() {
assertThat(metadata.getIndexMode(), equalTo(IndexMode.LOGSDB));
final IndexSettings settings = new IndexSettings(metadata, Settings.EMPTY);
assertThat(settings.getMode(), equalTo(IndexMode.LOGSDB));
assertThat("agent_id", equalTo(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey())));
assertThat(getIndexSetting(settings, IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey()), equalTo("agent_id"));
assertThat(settings.getIndexSortConfig().hasPrimarySortOnField("host.name"), equalTo(false));
assertThat(IndexMode.LOGSDB.getDefaultMapping(settings).string(), not(containsString("host")));
}

public void testSortMode() {
Expand Down
8 changes: 6 additions & 2 deletions test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,8 +302,12 @@ public void onRemoval(ShardId shardId, Accountable accountable) {}
mapperMetrics
);

if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping() != null) {
mapperService.merge(null, indexSettings.getMode().getDefaultMapping(), MapperService.MergeReason.MAPPING_UPDATE);
if (applyDefaultMapping && indexSettings.getMode().getDefaultMapping(indexSettings) != null) {
mapperService.merge(
null,
indexSettings.getMode().getDefaultMapping(indexSettings),
MapperService.MergeReason.MAPPING_UPDATE
);
}

return mapperService;
Expand Down
Loading

0 comments on commit 9bf6e3b

Please sign in to comment.