diff --git a/docs/changelog/112345.yaml b/docs/changelog/112345.yaml new file mode 100644 index 0000000000000..b922fe3754cbb --- /dev/null +++ b/docs/changelog/112345.yaml @@ -0,0 +1,8 @@ +pr: 112345 +summary: Allow dimension fields to have multiple values in standard and logsdb index + mode +area: Mapping +type: enhancement +issues: + - 112232 + - 112239 diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 96598ba38a3fe..8745b46fb5458 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -107,7 +107,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { @Override public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { - return new DocumentDimensions.OnlySingleValueAllowed(); + return DocumentDimensions.Noop.INSTANCE; } @Override @@ -281,7 +281,7 @@ public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { @Override public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { - return new DocumentDimensions.OnlySingleValueAllowed(); + return DocumentDimensions.Noop.INSTANCE; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java index aa69e4db50e76..f4995de2b080e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java @@ -12,8 +12,6 @@ import org.elasticsearch.index.IndexSettings; import java.net.InetAddress; -import java.util.HashSet; -import java.util.Set; /** * Collects dimensions from documents. @@ -49,59 +47,45 @@ default DocumentDimensions addString(String fieldName, String value) { DocumentDimensions validate(IndexSettings settings); /** - * Makes sure that each dimension only appears on time. + * Noop implementation that doesn't perform validations on dimension fields */ - class OnlySingleValueAllowed implements DocumentDimensions { - private final Set names = new HashSet<>(); + enum Noop implements DocumentDimensions { + + INSTANCE; @Override - public DocumentDimensions addString(String fieldName, BytesRef value) { - add(fieldName); + public DocumentDimensions addString(String fieldName, BytesRef utf8Value) { return this; } - // Override to skip the UTF-8 conversion that happens in the default implementation @Override public DocumentDimensions addString(String fieldName, String value) { - add(fieldName); return this; } @Override public DocumentDimensions addIp(String fieldName, InetAddress value) { - add(fieldName); return this; } @Override public DocumentDimensions addLong(String fieldName, long value) { - add(fieldName); return this; } @Override public DocumentDimensions addUnsignedLong(String fieldName, long value) { - add(fieldName); return this; } @Override public DocumentDimensions addBoolean(String fieldName, boolean value) { - add(fieldName); return this; } @Override - public DocumentDimensions validate(final IndexSettings settings) { - // DO NOTHING + public DocumentDimensions validate(IndexSettings settings) { return this; } - - private void add(String fieldName) { - boolean isNew = names.add(fieldName); - if (false == isNew) { - throw new IllegalArgumentException("Dimension field [" + fieldName + "] cannot be a multi-valued field."); - } - } - }; + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index e08a443bd74cb..03f030a26992d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -17,6 +17,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.script.BooleanFieldScript; @@ -25,11 +26,14 @@ import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; +import java.time.Instant; import java.util.List; import java.util.function.Function; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; public class BooleanFieldMapperTests extends MapperTestCase { @@ -257,17 +261,29 @@ public void testDimensionIndexedAndDocvalues() { } } - public void testDimensionMultiValuedField() throws IOException { - XContentBuilder mapping = fieldMapping(b -> { + public void testDimensionMultiValuedFieldTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - }); - DocumentMapper mapper = randomBoolean() ? createDocumentMapper(mapping) : createTimeSeriesModeDocumentMapper(mapping); + }), IndexMode.TIME_SERIES); Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", true, false)))); assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); } + public void testDimensionMultiValuedFieldNonTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true); + }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + + ParsedDocument doc = mapper.parse(source(b -> { + b.array("field", true, false); + b.field("@timestamp", Instant.now()); + })); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); + } + public void testDimensionInRoutingPath() throws IOException { MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("time_series_dimension", true))); IndexSettings settings = createIndexSettings( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index ba9c2e6c4a299..296871e258cd7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.script.IpFieldScript; @@ -26,6 +27,7 @@ import java.io.IOException; import java.net.InetAddress; +import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.function.Function; @@ -35,6 +37,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; public class IpFieldMapperTests extends MapperTestCase { @@ -255,11 +259,11 @@ public void testDimensionIndexedAndDocvalues() { } } - public void testDimensionMultiValuedField() throws IOException { + public void testDimensionMultiValuedFieldTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - })); + }), IndexMode.TIME_SERIES); Exception e = expectThrows( DocumentParsingException.class, @@ -268,6 +272,19 @@ public void testDimensionMultiValuedField() throws IOException { assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); } + public void testDimensionMultiValuedFieldNonTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true); + }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + + ParsedDocument doc = mapper.parse(source(b -> { + b.array("field", "192.168.1.1", "192.168.1.1"); + b.field("@timestamp", Instant.now()); + })); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); + } + @Override protected String generateRandomInputValue(MappedFieldType ft) { return NetworkAddress.format(randomIp(randomBoolean())); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 833b0a60827d0..d66575bc41cda 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -44,6 +45,7 @@ import org.elasticsearch.xcontent.XContentBuilder; import java.io.IOException; +import java.time.Instant; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -57,6 +59,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; public class KeywordFieldMapperTests extends MapperTestCase { @@ -373,17 +377,29 @@ public void testDimensionIndexedAndDocvalues() { } } - public void testDimensionMultiValuedField() throws IOException { - XContentBuilder mapping = fieldMapping(b -> { + public void testDimensionMultiValuedFieldTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - }); - DocumentMapper mapper = randomBoolean() ? createDocumentMapper(mapping) : createTimeSeriesModeDocumentMapper(mapping); + }), IndexMode.TIME_SERIES); Exception e = expectThrows(DocumentParsingException.class, () -> mapper.parse(source(b -> b.array("field", "1234", "45678")))); assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); } + public void testDimensionMultiValuedFieldNonTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true); + }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + + ParsedDocument doc = mapper.parse(source(b -> { + b.array("field", "1234", "45678"); + b.field("@timestamp", Instant.now()); + })); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); + } + public void testDimensionExtraLongKeyword() throws IOException { DocumentMapper mapper = createTimeSeriesModeDocumentMapper(fieldMapping(b -> { minimalMapping(b); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index aba20ec5d81c8..7b7044f528c89 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DocumentMapper; @@ -34,6 +35,7 @@ import org.junit.AssumptionViolatedException; import java.io.IOException; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -46,6 +48,8 @@ import static org.apache.lucene.tests.analysis.BaseTokenStreamTestCase.assertTokenStreamContents; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; public class FlattenedFieldMapperTests extends MapperTestCase { @@ -189,12 +193,11 @@ public void testDimensionIndexedAndDocvalues() { } } - public void testDimensionMultiValuedField() throws IOException { - XContentBuilder mapping = fieldMapping(b -> { + public void testDimensionMultiValuedFieldTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimensions", List.of("key1", "key2", "field3.key3")); - }); - DocumentMapper mapper = randomBoolean() ? createDocumentMapper(mapping) : createTimeSeriesModeDocumentMapper(mapping); + }), IndexMode.TIME_SERIES); Exception e = expectThrows( DocumentParsingException.class, @@ -203,6 +206,19 @@ public void testDimensionMultiValuedField() throws IOException { assertThat(e.getCause().getMessage(), containsString("Dimension field [field.key1] cannot be a multi-valued field")); } + public void testDimensionMultiValuedFieldNonTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimensions", List.of("key1", "key2", "field3.key3")); + }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + + ParsedDocument doc = mapper.parse(source(b -> { + b.array("field.key1", "value1", "value2"); + b.field("@timestamp", Instant.now()); + })); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); + } + public void testDisableIndex() throws Exception { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 7c11e7446e5c5..235bb7208fb08 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -139,6 +139,14 @@ protected static String randomIndexOptions() { return randomFrom("docs", "freqs", "positions", "offsets"); } + protected final DocumentMapper createDocumentMapper(XContentBuilder mappings, IndexMode indexMode) throws IOException { + return switch (indexMode) { + case STANDARD -> createDocumentMapper(mappings); + case TIME_SERIES -> createTimeSeriesModeDocumentMapper(mappings); + case LOGSDB -> createLogsModeDocumentMapper(mappings); + }; + } + protected final DocumentMapper createDocumentMapper(XContentBuilder mappings) throws IOException { return createMapperService(mappings).documentMapper(); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java index 99c500639bdde..4b12266196dee 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/WholeNumberFieldMapperTests.java @@ -9,11 +9,15 @@ package org.elasticsearch.index.mapper; import org.apache.lucene.index.IndexableField; +import org.elasticsearch.index.IndexMode; import java.io.IOException; +import java.time.Instant; import java.util.List; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; public abstract class WholeNumberFieldMapperTests extends NumberFieldMapperTests { @@ -69,11 +73,11 @@ public void testDimensionIndexedAndDocvalues() { } } - public void testDimensionMultiValuedField() throws IOException { + public void testDimensionMultiValuedFieldTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - })); + }), IndexMode.TIME_SERIES); Exception e = expectThrows( DocumentParsingException.class, @@ -82,6 +86,19 @@ public void testDimensionMultiValuedField() throws IOException { assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); } + public void testDimensionMultiValuedFieldNonTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true); + }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + + ParsedDocument doc = mapper.parse(source(b -> { + b.array("field", randomNumber(), randomNumber(), randomNumber()); + b.field("@timestamp", Instant.now()); + })); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); + } + public void testMetricAndDimension() { Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { minimalMapping(b); diff --git a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java index 753440cb0b789..46969d8dbb2ed 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java +++ b/x-pack/plugin/mapper-unsigned-long/src/test/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapperTests.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.math.BigInteger; +import java.time.Instant; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -38,6 +39,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesPattern; @@ -260,11 +263,11 @@ public void testDimensionIndexedAndDocvalues() { } } - public void testDimensionMultiValuedField() throws IOException { + public void testDimensionMultiValuedFieldTSDB() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { minimalMapping(b); b.field("time_series_dimension", true); - })); + }), IndexMode.TIME_SERIES); Exception e = expectThrows( DocumentParsingException.class, @@ -273,6 +276,19 @@ public void testDimensionMultiValuedField() throws IOException { assertThat(e.getCause().getMessage(), containsString("Dimension field [field] cannot be a multi-valued field")); } + public void testDimensionMultiValuedFieldNonTSDB() throws IOException { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> { + minimalMapping(b); + b.field("time_series_dimension", true); + }), randomFrom(IndexMode.STANDARD, IndexMode.LOGSDB)); + + ParsedDocument doc = mapper.parse(source(b -> { + b.array("field", randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()); + b.field("@timestamp", Instant.now()); + })); + assertThat(doc.docs().get(0).getFields("field"), hasSize(greaterThan(1))); + } + public void testMetricType() throws IOException { // Test default setting MapperService mapperService = createMapperService(fieldMapping(b -> minimalMapping(b))); diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml deleted file mode 100644 index d87c2a80deab8..0000000000000 --- a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs.tests.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -setup: - - do: - cluster.health: - wait_for_events: languid ---- -"Default data_stream.type must be logs": - - do: - bulk: - index: logs-generic.otel-default - refresh: true - body: - - create: {} - - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "attributes": { "foo": "bar"}, "body_text":"Error: Unable to connect to the database.","severity_text":"ERROR","severity_number":3,"trace_id":"abc123xyz456def789ghi012jkl345"}' - - is_false: errors - - do: - search: - index: logs-generic.otel-default - body: - fields: ["data_stream.type"] - - length: { hits.hits: 1 } - - match: { hits.hits.0.fields.data_stream\.type: ["logs"] } diff --git a/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml new file mode 100644 index 0000000000000..b0cf92b87667c --- /dev/null +++ b/x-pack/plugin/otel-data/src/yamlRestTest/resources/rest-api-spec/test/20_logs_tests.yml @@ -0,0 +1,53 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid +--- +"Default data_stream.type must be logs": + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - '{"@timestamp":"2024-07-18T14:48:33.467654000Z","data_stream":{"dataset":"generic.otel","namespace":"default"}, "attributes": { "foo": "bar"}, "body_text":"Error: Unable to connect to the database.","severity_text":"ERROR","severity_number":3,"trace_id":"abc123xyz456def789ghi012jkl345"}' + - is_false: errors + - do: + search: + index: logs-generic.otel-default + body: + fields: ["data_stream.type"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.data_stream\.type: ["logs"] } +--- +"Multi value fields": + - do: + bulk: + index: logs-generic.otel-default + refresh: true + body: + - create: {} + - "@timestamp": 2024-07-18T14:48:33.467654000Z + data_stream: + type: logs + dataset: generic.otel + namespace: default + resource: + attributes: + host.ip: ["127.0.0.1", "0.0.0.0"] + attributes: + foo: [3, 2, 1] + bar: [b, c, a] + body_text: "Error: Unable to connect to the database." + severity_text: ERROR + - is_false: errors + - do: + search: + index: logs-generic.otel-default + body: + fields: ["*"] + - length: { hits.hits: 1 } + - match: { hits.hits.0.fields.resource\.attributes\.host\.ip: ["0.0.0.0", "127.0.0.1"] } + - match: { hits.hits.0.fields.attributes\.foo: [1, 2, 3] } + - match: { hits.hits.0.fields.attributes\.bar: [a, b, c] }