Skip to content

Commit

Permalink
Allow synthetic source and disabled source for standard indices (elas…
Browse files Browse the repository at this point in the history 
…tic#114817)

When using the index.mapping.source.mode setting we need to make sure 
that it takes precedence and that is used also when standard index mode
is used. Without this patch we always return stored source if
_source.mode is not used and the setting is.

Relates elastic#114433
  • Loading branch information
@salvatore-campagna
salvatore-campagna authored Oct 15, 2024
1 parent e87b894 commit 3af4d67
Show file tree
Hide file tree
Showing 2 changed files with 239 additions and 20 deletions.
98 changes: 78 additions & 20 deletions server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,42 @@ public enum Mode {
true
);

private static final SourceFieldMapper DEFAULT_DISABLED = new SourceFieldMapper(
Mode.DISABLED,
Explicit.IMPLICIT_TRUE,
Strings.EMPTY_ARRAY,
Strings.EMPTY_ARRAY,
null,
true
);

private static final SourceFieldMapper DEFAULT_DISABLED_NO_RECOVERY_SOURCE = new SourceFieldMapper(
Mode.DISABLED,
Explicit.IMPLICIT_TRUE,
Strings.EMPTY_ARRAY,
Strings.EMPTY_ARRAY,
null,
false
);

private static final SourceFieldMapper DEFAULT_SYNTHETIC = new SourceFieldMapper(
Mode.SYNTHETIC,
Explicit.IMPLICIT_TRUE,
Strings.EMPTY_ARRAY,
Strings.EMPTY_ARRAY,
null,
true
);

private static final SourceFieldMapper DEFAULT_SYNTHETIC_NO_RECOVERY_SOURCE = new SourceFieldMapper(
Mode.SYNTHETIC,
Explicit.IMPLICIT_TRUE,
Strings.EMPTY_ARRAY,
Strings.EMPTY_ARRAY,
null,
false
);

private static final SourceFieldMapper DEFAULT_NO_RECOVERY_SOURCE = new SourceFieldMapper(
null,
Explicit.IMPLICIT_TRUE,
Expand Down Expand Up @@ -297,7 +333,7 @@ public SourceFieldMapper build() {
? INDEX_MAPPER_SOURCE_MODE_SETTING.get(settings)
: mode.get();
if (isDefault(sourceMode)) {
return resolveSourceMode(indexMode, sourceMode, enableRecoverySource);
return resolveSourceMode(indexMode, sourceMode == null ? Mode.STORED : sourceMode, enableRecoverySource);

}
if (supportsNonDefaultParameterValues == false) {
Expand Down Expand Up @@ -340,25 +376,39 @@ public SourceFieldMapper build() {
}

private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, final Mode sourceMode, boolean enableRecoverySource) {
if (indexMode == IndexMode.STANDARD) {
return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
switch (indexMode) {
case STANDARD:
switch (sourceMode) {
case SYNTHETIC:
return enableRecoverySource ? DEFAULT_SYNTHETIC : DEFAULT_SYNTHETIC_NO_RECOVERY_SOURCE;
case STORED:
return enableRecoverySource ? DEFAULT : DEFAULT_NO_RECOVERY_SOURCE;
case DISABLED:
return enableRecoverySource ? DEFAULT_DISABLED : DEFAULT_DISABLED_NO_RECOVERY_SOURCE;
default:
throw new IllegalArgumentException("Unsupported source mode: " + sourceMode);
}
case TIME_SERIES:
case LOGSDB:
switch (sourceMode) {
case SYNTHETIC:
return enableRecoverySource
? (indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT)
: (indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_NO_RECOVERY_SOURCE : LOGSDB_DEFAULT_NO_RECOVERY_SOURCE);
case STORED:
return enableRecoverySource
? (indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED)
: (indexMode == IndexMode.TIME_SERIES
? TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED);
case DISABLED:
throw new IllegalArgumentException("_source can not be disabled in index using [" + indexMode + "] index mode");
default:
throw new IllegalArgumentException("Unsupported source mode: " + sourceMode);
}
default:
throw new IllegalArgumentException("Unsupported index mode: " + indexMode);
}
final SourceFieldMapper syntheticWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES
? TSDB_DEFAULT_NO_RECOVERY_SOURCE
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE;
final SourceFieldMapper syntheticWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT : LOGSDB_DEFAULT;
final SourceFieldMapper storedWithoutRecoverySource = indexMode == IndexMode.TIME_SERIES
? TSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED
: LOGSDB_DEFAULT_NO_RECOVERY_SOURCE_STORED;
final SourceFieldMapper storedWithRecoverySource = indexMode == IndexMode.TIME_SERIES ? TSDB_DEFAULT_STORED : LOGSDB_DEFAULT_STORED;

return switch (sourceMode) {
case SYNTHETIC -> enableRecoverySource ? syntheticWithRecoverySource : syntheticWithoutRecoverySource;
case STORED -> enableRecoverySource ? storedWithRecoverySource : storedWithoutRecoverySource;
case DISABLED -> throw new IllegalArgumentException(
"_source can not be disabled in index using [" + indexMode + "] index mode"
);
};
}

public static final TypeParser PARSER = new ConfigurableTypeParser(c -> {
Expand All @@ -371,7 +421,7 @@ private static SourceFieldMapper resolveSourceMode(final IndexMode indexMode, fi
return enableRecoverySource ? TSDB_LEGACY_DEFAULT : TSDB_LEGACY_DEFAULT_NO_RECOVERY_SOURCE;
}
}
return resolveSourceMode(indexMode, settingSourceMode, enableRecoverySource);
return resolveSourceMode(indexMode, settingSourceMode == null ? Mode.STORED : settingSourceMode, enableRecoverySource);
},
c -> new Builder(
c.getIndexSettings().getMode(),
Expand Down Expand Up @@ -541,4 +591,12 @@ public SourceLoader newSourceLoader(Mapping mapping, SourceFieldMetrics metrics)
public boolean isSynthetic() {
return mode == Mode.SYNTHETIC;
}

public boolean isDisabled() {
return mode == Mode.DISABLED;
}

public boolean isStored() {
return mode == null || mode == Mode.STORED;
}
}
161 changes: 161 additions & 0 deletions server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,6 +443,167 @@ public void testRecoverySourceWithLogs() throws IOException {
}
}

public void testStandardIndexModeWithSourceModeSetting() throws IOException {
// Test for IndexMode.STANDARD
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
.build();
final MapperService mapperService = createMapperService(settings, mappings);
DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isSynthetic());
}
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
.build();
final MapperService mapperService = createMapperService(settings, mappings);
final DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isStored());
}
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
.build();
final MapperService mapperService = createMapperService(settings, mappings);
final DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isDisabled());
}

// Test for IndexMode.LOGSDB
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
.build();
final MapperService mapperService = createMapperService(settings, mappings);
DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isSynthetic());
}
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
.build();
final MapperService mapperService = createMapperService(settings, mappings);
final DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isStored());
}
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
.build();
var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings));
assertEquals("Failed to parse mapping: _source can not be disabled in index using [logsdb] index mode", ex.getMessage());
}

// Test for IndexMode.TIME_SERIES
{
final String mappings = """
{
"_doc" : {
"properties": {
"routing_field": {
"type": "keyword",
"time_series_dimension": true
}
}
}
}
""";
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
.build();
final MapperService mapperService = createMapperService(settings, mappings);
DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isSynthetic());
}
{
final String mappings = """
{
"_doc" : {
"properties": {
"routing_field": {
"type": "keyword",
"time_series_dimension": true
}
}
}
}
""";
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
.build();
final MapperService mapperService = createMapperService(settings, mappings);
final DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isStored());
}
{
final String mappings = """
{
"_doc" : {
"properties": {
"routing_field": {
"type": "keyword",
"time_series_dimension": true
}
}
}
}
""";
final Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name())
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing_field")
.build();
var ex = expectThrows(MapperParsingException.class, () -> createMapperService(settings, mappings));
assertEquals("Failed to parse mapping: _source can not be disabled in index using [time_series] index mode", ex.getMessage());
}

// Test cases without IndexMode (default to standard)
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
.build();
final MapperService mapperService = createMapperService(settings, mappings);
DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isSynthetic());
}
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.STORED)
.build();
final MapperService mapperService = createMapperService(settings, mappings);
final DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isStored());
}
{
final XContentBuilder mappings = topMapping(b -> {});
final Settings settings = Settings.builder()
.put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.DISABLED)
.build();
final MapperService mapperService = createMapperService(settings, mappings);
final DocumentMapper docMapper = mapperService.documentMapper();
assertTrue(docMapper.sourceMapper().isDisabled());
}
}

public void testRecoverySourceWithLogsCustom() throws IOException {
XContentBuilder mappings = topMapping(b -> b.startObject(SourceFieldMapper.NAME).field("mode", "synthetic").endObject());
{
Expand Down

0 comments on commit 3af4d67

Please sign in to comment.