From 7aa71c54e74bdd57779824a4ac0a887e871271ac Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 1 Feb 2024 16:14:12 +0200 Subject: [PATCH 01/51] Fix test failure https://gradle-enterprise.elastic.co/s/icg66i6mwnjoi --- .../resources/rest-api-spec/test/data_stream/150_tsdb.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index c49775d9710be..a7fd855e5b2ac 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -196,6 +196,7 @@ index without timestamp with pipeline: dynamic templates: - skip: version: " - 8.12.99" + features: "default_shards" reason: "Support for dynamic fields was added in 8.13" - do: indices.put_index_template: @@ -287,6 +288,7 @@ dynamic templates: dynamic templates - conflicting aliases: - skip: version: " - 8.12.99" + features: "default_shards" reason: "Support for dynamic fields was added in 8.13" - do: indices.put_index_template: From bf946a94f30bc49c5dc2d9fd09474498cd98234a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 1 Feb 2024 16:34:15 +0200 Subject: [PATCH 02/51] Fix test failure https://gradle-enterprise.elastic.co/s/icg66i6mwnjoi --- .../resources/rest-api-spec/test/data_stream/150_tsdb.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index a7fd855e5b2ac..794b27612fdd8 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -262,7 +262,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: { "attributes.another.dim": "C", "attributes.dim": "A" } } + - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur4gYpiKF9nXxRUMHnJBiqqP58deEIun8H8" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -281,7 +281,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: { "attributes.another.dim": "C", "attributes.dim": "A" } } + - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur4gYpiKF9nXxRUMHnJBiqqP58deEIun8H8" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -358,7 +358,7 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: { "resource_attributes.dim": "C", "attributes.dim": "A" } } + - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -377,7 +377,7 @@ dynamic templates - conflicting aliases: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: { "resource_attributes.dim": "C", "attributes.dim": "A" } } + - match: { aggregations.filterA.tsids.buckets.0.key: "KGejYryCnrIkXYZdIF_Q8F8X2dfFIGKYisFh7t1RGGWOWgWU7C0RiFE" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- From a3a2ac00430e45627335767129b592362fe3bbff Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 2 Feb 2024 14:41:13 +0200 Subject: [PATCH 03/51] Nest pass-through objects within objects --- .../test/data_stream/150_tsdb.yml | 136 ++++++++++++--- .../index/mapper/ObjectMapper.java | 23 +-- .../index/mapper/PassThroughObjectMapper.java | 9 +- .../index/mapper/RootObjectMapper.java | 18 +- .../index/mapper/DynamicTemplatesTests.java | 2 +- .../index/mapper/ObjectMapperTests.java | 4 +- .../index/mapper/RootObjectMapperTests.java | 156 ++++++++++++++---- 7 files changed, 270 insertions(+), 78 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 794b27612fdd8..81ede2b045e61 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -381,12 +381,12 @@ dynamic templates - conflicting aliases: - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- -dynamic templates - subobject in passthrough object error: +dynamic templates with nesting: - skip: version: " - 8.12.99" + features: "default_shards" reason: "Support for dynamic fields was added in 8.13" - do: - catch: /Tried to add subobject \[subcategory\] to object \[attributes\] which does not support subobjects/ indices.put_index_template: name: my-dynamic-template body: @@ -395,21 +395,116 @@ dynamic templates - subobject in passthrough object error: template: settings: index: + number_of_shards: 1 mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z mappings: properties: attributes: type: passthrough + dynamic: true + time_series_dimension: true + resource: + type: object properties: - subcategory: - type: object - properties: - dim: - type: keyword + attributes: + type: passthrough + dynamic: true + time_series_dimension: true + dynamic_templates: + - counter_metric: + mapping: + type: integer + time_series_metric: counter - do: - catch: /Mapping definition for \[attributes\] has unsupported parameters:\ \[subobjects \:\ true\]/ + bulk: + index: k9s + refresh: true + body: + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim": "A", "resource.attributes.another.dim": "C", "attributes.more.dim": "E" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim": "A", "resource.attributes.another.dim": "C", "attributes.more.dim": "E" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim": "B", "resource.attributes.another.dim": "D", "attributes.more.dim": "F" }' + - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim": "B", "resource.attributes.another.dim": "D", "attributes.more.dim": "F" }' + + - do: + search: + index: k9s + body: + size: 0 + + - match: { hits.total.value: 4 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + dim: A + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + another.dim: C + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + more.dim: E + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + +--- +dynamic templates - subobject in passthrough object error: + - skip: + version: " - 8.12.99" + reason: "Support for dynamic fields was added in 8.13" + - do: + catch: /Tried to add subobject \[subcategory\] to object \[attributes\] which does not support subobjects/ indices.put_index_template: name: my-dynamic-template body: @@ -418,24 +513,21 @@ dynamic templates - subobject in passthrough object error: template: settings: index: - number_of_shards: 1 mode: time_series - time_series: - start_time: 2023-08-31T13:03:08.138Z mappings: properties: attributes: type: passthrough - subobjects: true + properties: + subcategory: + type: object + properties: + dim: + type: keyword ---- -dynamic templates - passthrough not under root error: - - skip: - version: " - 8.12.99" - reason: "Support for dynamic fields was added in 8.13" - do: - catch: /Tried to add passthrough subobject \[attributes\] to object \[resource\], passthrough is not supported as a subobject/ + catch: /Mapping definition for \[attributes\] has unsupported parameters:\ \[subobjects \:\ true\]/ indices.put_index_template: name: my-dynamic-template body: @@ -444,11 +536,13 @@ dynamic templates - passthrough not under root error: template: settings: index: + number_of_shards: 1 mode: time_series + time_series: + start_time: 2023-08-31T13:03:08.138Z mappings: properties: - "resource.attributes": + attributes: type: passthrough - dynamic: true - time_series_dimension: true + subobjects: true diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 1ed9713d73e75..9d7353859ed25 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -295,7 +295,10 @@ protected static void parseProperties( } } - if (objBuilder.subobjects.value() == false && type.equals(ObjectMapper.CONTENT_TYPE)) { + if (objBuilder.subobjects.value() == false + && (type.equals(ObjectMapper.CONTENT_TYPE) + || type.equals(NestedObjectMapper.CONTENT_TYPE) + || type.equals(PassThroughObjectMapper.CONTENT_TYPE))) { throw new MapperParsingException( "Tried to add subobject [" + fieldName @@ -304,24 +307,6 @@ protected static void parseProperties( + "] which does not support subobjects" ); } - if (objBuilder.subobjects.value() == false && type.equals(NestedObjectMapper.CONTENT_TYPE)) { - throw new MapperParsingException( - "Tried to add nested object [" - + fieldName - + "] to object [" - + objBuilder.name() - + "] which does not support subobjects" - ); - } - if (type.equals(PassThroughObjectMapper.CONTENT_TYPE) && objBuilder instanceof RootObjectMapper.Builder == false) { - throw new MapperParsingException( - "Tried to add passthrough subobject [" - + fieldName - + "] to object [" - + objBuilder.name() - + "], passthrough is not supported as a subobject" - ); - } Mapper.TypeParser typeParser = parserContext.typeParser(type); if (typeParser == null) { throw new MapperParsingException("No handler for type [" + type + "] declared on field [" + fieldName + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index 7688b217ab7fc..b49c9328fcc79 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -57,6 +57,7 @@ public PassThroughObjectMapper.Builder setContainsDimensions() { public PassThroughObjectMapper build(MapperBuilderContext context) { return new PassThroughObjectMapper( name, + context.buildFullName(name), enabled, dynamic, buildMappers(context.createChildContext(name)), @@ -70,19 +71,20 @@ public PassThroughObjectMapper build(MapperBuilderContext context) { PassThroughObjectMapper( String name, + String fullPath, Explicit enabled, Dynamic dynamic, Map mappers, Explicit timeSeriesDimensionSubFields ) { // Subobjects are not currently supported. - super(name, name, enabled, Explicit.IMPLICIT_FALSE, dynamic, mappers); + super(name, fullPath, enabled, Explicit.IMPLICIT_FALSE, dynamic, mappers); this.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; } @Override PassThroughObjectMapper withoutMappers() { - return new PassThroughObjectMapper(simpleName(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields); + return new PassThroughObjectMapper(simpleName(), fullPath(), enabled, dynamic, Map.of(), timeSeriesDimensionSubFields); } public boolean containsDimensions() { @@ -91,7 +93,7 @@ public boolean containsDimensions() { @Override public PassThroughObjectMapper.Builder newBuilder(IndexVersion indexVersionCreated) { - PassThroughObjectMapper.Builder builder = new PassThroughObjectMapper.Builder(name()); + PassThroughObjectMapper.Builder builder = new PassThroughObjectMapper.Builder(simpleName()); builder.enabled = enabled; builder.dynamic = dynamic; builder.timeSeriesDimensionSubFields = timeSeriesDimensionSubFields; @@ -108,6 +110,7 @@ public PassThroughObjectMapper merge(ObjectMapper mergeWith, MergeReason reason, return new PassThroughObjectMapper( simpleName(), + fullPath(), mergeResult.enabled(), mergeResult.dynamic(), mergeResult.mappers(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 86bdb2aa2bba7..7994c018f40f2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -45,6 +45,7 @@ public class RootObjectMapper extends ObjectMapper { private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(RootObjectMapper.class); + private static final int MAX_NESTING_LEVEL_FOR_PASS_THROUGH_OBJECTS = 20; /** * Parameter used when serializing {@link RootObjectMapper} and request that the runtime section is skipped. @@ -111,7 +112,11 @@ public RootObjectMapper.Builder addRuntimeFields(Map runti @Override public RootObjectMapper build(MapperBuilderContext context) { Map mappers = buildMappers(context); - mappers.putAll(getAliasMappers(mappers, context)); + + Map aliasMappers = new HashMap<>(); + getAliasMappers(mappers, aliasMappers, context, 0); + mappers.putAll(aliasMappers); + return new RootObjectMapper( name, enabled, @@ -126,8 +131,11 @@ public RootObjectMapper build(MapperBuilderContext context) { ); } - Map getAliasMappers(Map mappers, MapperBuilderContext context) { - Map aliasMappers = new HashMap<>(); + void getAliasMappers(Map mappers, Map aliasMappers, MapperBuilderContext context, int level) { + if (level >= MAX_NESTING_LEVEL_FOR_PASS_THROUGH_OBJECTS) { + logger.warn("Exceeded maximum nesting level for searching for pass-through object fields within object fields."); + return; + } for (Mapper mapper : mappers.values()) { // Create aliases for all fields in child passthrough mappers and place them under the root object. if (mapper instanceof PassThroughObjectMapper passthroughMapper) { @@ -154,9 +162,11 @@ Map getAliasMappers(Map mappers, MapperBuilderCo } } } + } else if (mapper instanceof ObjectMapper objectMapper) { + // Call recursively to check child fields. The level guards against long recursive call sequences. + getAliasMappers(objectMapper.mappers, aliasMappers, context, level + 1); } } - return aliasMappers; } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java index 54db5832c2726..10bd6c667c26e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicTemplatesTests.java @@ -1391,7 +1391,7 @@ public void testSubobjectsFalseWithInnerNestedFromDynamicTemplate() { ); assertThat(exception.getRootCause(), instanceOf(MapperParsingException.class)); assertEquals( - "Tried to add nested object [time] to object [__dynamic__test] which does not support subobjects", + "Tried to add subobject [time] to object [__dynamic__test] which does not support subobjects", exception.getRootCause().getMessage() ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java index 6e958ddbea904..cbb0929b813fc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperTests.java @@ -407,7 +407,7 @@ public void testSubobjectsFalseWithInnerNested() { b.endObject(); }))); assertEquals( - "Failed to parse mapping: Tried to add nested object [time] to object [service] which does not support subobjects", + "Failed to parse mapping: Tried to add subobject [time] to object [service] which does not support subobjects", exception.getMessage() ); } @@ -457,7 +457,7 @@ public void testSubobjectsFalseRootWithInnerNested() { b.endObject(); }))); assertEquals( - "Failed to parse mapping: Tried to add nested object [metrics.service] to object [_doc] which does not support subobjects", + "Failed to parse mapping: Tried to add subobject [metrics.service] to object [_doc] which does not support subobjects", exception.getMessage() ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java index b77019806fc4f..55afc8be44380 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RootObjectMapperTests.java @@ -360,50 +360,150 @@ public void testPassThroughObjectWithAliases() throws IOException { assertThat(mapperService.mappingLookup().getMapper("labels.dim"), instanceOf(KeywordFieldMapper.class)); } + public void testPassThroughObjectNested() throws IOException { + MapperService mapperService = createMapperService(mapping(b -> { + b.startObject("resource").field("type", "object"); + { + b.startObject("properties"); + { + b.startObject("attributes").field("type", "passthrough"); + { + b.startObject("properties"); + b.startObject("dim").field("type", "keyword").endObject(); + b.endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endObject(); + })); + assertThat(mapperService.mappingLookup().getMapper("dim"), instanceOf(FieldAliasMapper.class)); + assertThat(mapperService.mappingLookup().getMapper("resource.attributes.dim"), instanceOf(KeywordFieldMapper.class)); + } + public void testAliasMappersCreatesAlias() throws Exception { var context = MapperBuilderContext.root(false, false); - - Map fields = new HashMap<>(); - fields.put("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)); - - Map mappers = new HashMap<>(); - mappers.put( - "labels", - new PassThroughObjectMapper("labels", Explicit.EXPLICIT_TRUE, ObjectMapper.Dynamic.FALSE, fields, Explicit.EXPLICIT_FALSE) + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "labels", + new PassThroughObjectMapper( + "labels", + "labels", + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)), + Explicit.EXPLICIT_FALSE + ) + ), + aliases, + context, + 0 ); - - Map aliases = new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers(mappers, context); assertEquals(1, aliases.size()); assertThat(aliases.get("host"), instanceOf(FieldAliasMapper.class)); } - public void testAliasMappersCreatesNoAliasForRegularObject() throws Exception { + public void testAliasMappersCreatesAliasNested() throws Exception { var context = MapperBuilderContext.root(false, false); + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "outer", + new ObjectMapper( + "outer", + "outer", + Explicit.EXPLICIT_TRUE, + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of( + "inner", + new PassThroughObjectMapper( + "inner", + "outer.inner", + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)), + Explicit.EXPLICIT_FALSE + ) + ) + ) + ), + aliases, + context, + 0 + ); + assertEquals(1, aliases.size()); + assertThat(aliases.get("host"), instanceOf(FieldAliasMapper.class)); + } - Map fields = new HashMap<>(); - fields.put("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)); + public void testAliasMappersExitsInDeepNesting() throws Exception { + var context = MapperBuilderContext.root(false, false); + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "labels", + new PassThroughObjectMapper( + "labels", + "labels", + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)), + Explicit.EXPLICIT_FALSE + ) + ), + aliases, + context, + 1_000_000 + ); + assertTrue(aliases.isEmpty()); + } - Map mappers = new HashMap<>(); - mappers.put( - "labels", - new ObjectMapper("labels", "labels", Explicit.EXPLICIT_TRUE, Explicit.EXPLICIT_FALSE, ObjectMapper.Dynamic.FALSE, fields) + public void testAliasMappersCreatesNoAliasForRegularObject() throws Exception { + var context = MapperBuilderContext.root(false, false); + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "labels", + new ObjectMapper( + "labels", + "labels", + Explicit.EXPLICIT_TRUE, + Explicit.EXPLICIT_FALSE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)) + ) + ), + aliases, + context, + 0 ); - assertTrue(new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers(mappers, context).isEmpty()); + assertTrue(aliases.isEmpty()); } public void testAliasMappersConflictingField() throws Exception { var context = MapperBuilderContext.root(false, false); - - Map fields = new HashMap<>(); - fields.put("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)); - - Map mappers = new HashMap<>(); - mappers.put( - "labels", - new PassThroughObjectMapper("labels", Explicit.EXPLICIT_TRUE, ObjectMapper.Dynamic.FALSE, fields, Explicit.EXPLICIT_FALSE) + Map aliases = new HashMap<>(); + new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers( + Map.of( + "labels", + new PassThroughObjectMapper( + "labels", + "labels", + Explicit.EXPLICIT_TRUE, + ObjectMapper.Dynamic.FALSE, + Map.of("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)), + Explicit.EXPLICIT_FALSE + ), + "host", + new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context) + ), + aliases, + context, + 0 ); - mappers.put("host", new KeywordFieldMapper.Builder("host", IndexVersion.current()).build(context)); - assertTrue(new RootObjectMapper.Builder("root", Explicit.EXPLICIT_FALSE).getAliasMappers(mappers, context).isEmpty()); + assertTrue(aliases.isEmpty()); } public void testEmptyType() throws Exception { From beea7344b6470708000ab34e2b3adc300bf7c0fe Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Fri, 2 Feb 2024 17:20:25 +0200 Subject: [PATCH 04/51] Support numeric fields as pass-through dimensions --- .../test/data_stream/150_tsdb.yml | 10 +- .../index/mapper/IpFieldMapper.java | 22 ++++ .../index/mapper/NumberFieldMapper.java | 22 ++++ .../index/mapper/PassThroughObjectMapper.java | 10 +- .../mapper/TsidExtractingIdFieldMapper.java | 14 --- .../index/TimeSeriesModeTests.java | 14 +-- .../index/mapper/KeywordFieldMapperTests.java | 9 -- .../index/mapper/MapperTestCase.java | 114 +++++++++--------- .../unsignedlong/UnsignedLongFieldMapper.java | 13 ++ 9 files changed, 130 insertions(+), 98 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 81ede2b045e61..0b60993e60e91 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -230,13 +230,13 @@ dynamic templates: refresh: true body: - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "attributes.another.dim": "C" }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "attributes.another.dim": 10 }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "attributes.another.dim": "C" }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "attributes.another.dim": 10 }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "attributes.another.dim": "D" }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "attributes.another.dim": 20 }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "attributes.another.dim": "D" }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "attributes.another.dim": 20 }' - do: search: @@ -274,7 +274,7 @@ dynamic templates: filterA: filter: term: - another.dim: C + another.dim: 10 aggs: tsids: terms: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 8ce726b49ff66..7b1e30ef3c647 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -468,6 +468,28 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi public boolean isDimension() { return isDimension; } + + public void validateMatchedRoutingPath(final String routingPath) { + if (false == isDimension) { + throw new IllegalArgumentException( + "All fields that match routing_path " + + "must be keywords with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] was not a dimension." + ); + } + if (scriptValues != null) { + throw new IllegalArgumentException( + "All fields that match routing_path must be keywords with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] has a [script] parameter." + ); + } + } } private final boolean indexed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 5935eaf2c3d14..e1ac33c132d30 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1759,6 +1759,28 @@ public boolean isDimension() { public MetricType getMetricType() { return metricType; } + + public void validateMatchedRoutingPath(final String routingPath) { + if (false == isDimension) { + throw new IllegalArgumentException( + "All fields that match routing_path " + + "must be keywords with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] was not a dimension." + ); + } + if (scriptValues != null) { + throw new IllegalArgumentException( + "All fields that match routing_path must be keywords with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] has a [script] parameter." + ); + } + } } private final NumberType type; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index b49c9328fcc79..daa14b1eea79c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -41,8 +41,14 @@ public Builder(String name) { @Override public PassThroughObjectMapper.Builder add(Mapper.Builder builder) { - if (timeSeriesDimensionSubFields.value() && builder instanceof KeywordFieldMapper.Builder keywordBuilder) { - keywordBuilder.dimension(true); + if (timeSeriesDimensionSubFields.value()) { + if (builder instanceof KeywordFieldMapper.Builder keywordBuilder) { + keywordBuilder.dimension(true); + } else if (builder instanceof NumberFieldMapper.Builder numberBuilder) { + numberBuilder.dimension(true); + } else if (builder instanceof IpFieldMapper.Builder ipBuilder) { + ipBuilder.dimension(true); + } } super.add(builder); return this; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 27e281ed7fb52..ad8bc71d5f8a0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -116,20 +116,6 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra long timestamp = timestampFields.get(0).numericValue().longValue(); byte[] suffix = new byte[16]; String id = createId(context.hasDynamicMappers() == false, routingBuilder, tsid, timestamp, suffix); - /* - * Make sure that _id from extracting the tsid matches that _id - * from extracting the _source. This should be true for all valid - * documents with valid mappings. *But* some invalid mappings - * will not parse the field but be rejected later by the dynamic - * mappings machinery. So if there are any dynamic mappings - * at all we just skip the assertion because we can't be sure - * it always must pass. - */ - IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); - assert context.getDynamicMappers().isEmpty() == false - || context.getDynamicRuntimeFields().isEmpty() == false - || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); - if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { throw new IllegalArgumentException( String.format( diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index fc29d13667d33..847bb1dfbfb0b 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -199,22 +199,14 @@ public void testRoutingPathMatchesNonDimensionKeyword() { ); } - public void testRoutingPathMatchesNonKeyword() { + public void testRoutingPathMatchesNonKeyword() throws IOException { Settings s = getSettings(randomBoolean() ? "dim.non_kwd" : "dim.*"); - Exception e = expectThrows(IllegalArgumentException.class, () -> createMapperService(s, mapping(b -> { + createMapperService(s, mapping(b -> { b.startObject("dim").startObject("properties"); b.startObject("non_kwd").field("type", "integer").field("time_series_dimension", true).endObject(); b.startObject("dim").field("type", "keyword").field("time_series_dimension", true).endObject(); b.endObject().endObject(); - }))); - assertThat( - e.getMessage(), - equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [dim.non_kwd] was [integer]." - ) - ); + })); } public void testRoutingPathMatchesScriptedKeyword() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 546551baf4408..07134fd3aa7b1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -599,15 +599,6 @@ protected boolean dedupAfterFetch() { return true; } - @Override - protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - return "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + mapper.name() - + "] was not a dimension."; - } - public void testDimensionInRoutingPath() throws IOException { MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("time_series_dimension", true))); IndexSettings settings = createIndexSettings( diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 05aee30799de2..a6a97b21c80ed 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1025,72 +1025,72 @@ public final void testMinimalIsInvalidInRoutingPath() throws IOException { } } - protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - return "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + mapper.name() - + "] was [" - + mapper.typeName() - + "]."; - } - - public record SyntheticSourceExample( - CheckedConsumer inputValue, - CheckedConsumer expectedForSyntheticSource, - CheckedConsumer expectedForBlockLoader, - CheckedConsumer mapping - ) { - public SyntheticSourceExample(Object inputValue, Object result, CheckedConsumer mapping) { - this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(result), mapping); + String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { + if (mapper instanceof KeywordFieldMapper || mapper instanceof NumberFieldMapper || mapper instanceof IpFieldMapper) { + return "All fields that match routing_path must be keywords with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + mapper.name() + + "] was not a dimension."; } + return"All fields that match routing_path must be keywords with [time_series_dimension: true] "+"or flattened fields with a list of dimensions in [time_series_dimensions] and "+"without the [script] parameter. ["+mapper.name()+"] was ["+mapper.typeName()+"]."; +} - /** - * Create an example that returns different results from doc values - * than from synthetic source. - */ - public SyntheticSourceExample( - Object inputValue, - Object result, - Object blockLoaderResults, - CheckedConsumer mapping - ) { - this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(blockLoaderResults), mapping); - } +public record SyntheticSourceExample( + CheckedConsumer inputValue, + CheckedConsumer expectedForSyntheticSource, + CheckedConsumer expectedForBlockLoader, + CheckedConsumer mapping +) { + public SyntheticSourceExample(Object inputValue, Object result, CheckedConsumer mapping) { + this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(result), mapping); + } - private void buildInput(XContentBuilder b) throws IOException { - b.field("field"); - inputValue.accept(b); - } + /** + * Create an example that returns different results from doc values + * than from synthetic source. + */ + public SyntheticSourceExample( + Object inputValue, + Object result, + Object blockLoaderResults, + CheckedConsumer mapping + ) { + this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(blockLoaderResults), mapping); + } - private String expected() throws IOException { - XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); - expectedForSyntheticSource.accept(b); - return Strings.toString(b.endObject()); - } + private void buildInput(XContentBuilder b) throws IOException { + b.field("field"); + inputValue.accept(b); + } - private Object expectedParsedForBlockLoader() throws IOException { - XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); - expectedForBlockLoader.accept(b); - String str = Strings.toString(b.endObject()); - return XContentHelper.convertToMap(JsonXContent.jsonXContent, str, false).get("field"); - } + private String expected() throws IOException { + XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); + expectedForSyntheticSource.accept(b); + return Strings.toString(b.endObject()); } - public record SyntheticSourceInvalidExample(Matcher error, CheckedConsumer mapping) {} + private Object expectedParsedForBlockLoader() throws IOException { + XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); + expectedForBlockLoader.accept(b); + String str = Strings.toString(b.endObject()); + return XContentHelper.convertToMap(JsonXContent.jsonXContent, str, false).get("field"); + } +} - public interface SyntheticSourceSupport { - /** - * Examples that should work when source is generated from doc values. - */ - SyntheticSourceExample example(int maxValues) throws IOException; +public record SyntheticSourceInvalidExample(Matcher error, CheckedConsumer mapping) {} - /** - * Examples of mappings that should be rejected when source is configured to - * be loaded from doc values. - */ - List invalidExample() throws IOException; - } +public interface SyntheticSourceSupport { + /** + * Examples that should work when source is generated from doc values. + */ + SyntheticSourceExample example(int maxValues) throws IOException; + + /** + * Examples of mappings that should be rejected when source is configured to + * be loaded from doc values. + */ + List invalidExample() throws IOException;} protected abstract SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index c468d7bcd6718..ded6e135ad0f1 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -553,6 +553,19 @@ public boolean isDimension() { public MetricType getMetricType() { return metricType; } + + public void validateMatchedRoutingPath(final String routingPath) { + if (false == isDimension) { + throw new IllegalArgumentException( + "All fields that match routing_path " + + "must be keywords with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] was not a dimension." + ); + } + } } private final boolean indexed; From 2e18f8befe28362d729f12439b64585e6ab34564 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Fri, 2 Feb 2024 17:31:19 +0200 Subject: [PATCH 05/51] Update docs/changelog/105073.yaml --- docs/changelog/105073.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/105073.yaml diff --git a/docs/changelog/105073.yaml b/docs/changelog/105073.yaml new file mode 100644 index 0000000000000..62b22f17cf4ea --- /dev/null +++ b/docs/changelog/105073.yaml @@ -0,0 +1,5 @@ +pr: 105073 +summary: Support non-keyword fields as dimension subfields of pass-through object +area: TSDB +type: enhancement +issues: [] From cb94c8a1ef037ef886438cd570dce3e60ae94bc7 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 5 Feb 2024 10:16:09 +0200 Subject: [PATCH 06/51] fix tests --- .../resources/rest-api-spec/test/data_stream/150_tsdb.yml | 4 ++-- .../index/mapper/extras/TokenCountFieldMapperTests.java | 5 +++++ .../elasticsearch/index/mapper/IpFieldMapperTests.java | 5 +++++ .../index/mapper/KeywordFieldMapperTests.java | 5 +++++ .../org/elasticsearch/index/mapper/MapperTestCase.java | 8 ++++++-- .../index/mapper/NumberFieldMapperTests.java | 5 +++++ 6 files changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 0b60993e60e91..437bae15cdfc9 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -262,7 +262,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur4gYpiKF9nXxRUMHnJBiqqP58deEIun8H8" } + - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur7qNQVbF9nXxeO_bMXMxPMOBKO-Urpf2SU" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -281,7 +281,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur4gYpiKF9nXxRUMHnJBiqqP58deEIun8H8" } + - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur7qNQVbF9nXxeO_bMXMxPMOBKO-Urpf2SU" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java index 1636def53536b..efbf246a32f09 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java @@ -62,6 +62,11 @@ protected Object getSampleValueForQuery() { return 1; } + @Override + protected boolean fieldMapperSupportsDimension() { + return true; + } + @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("index", b -> b.field("index", false)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index ba9c2e6c4a299..f06bc457ec9bb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -43,6 +43,11 @@ protected Object getSampleValueForDocument() { return "::1"; } + @Override + protected boolean fieldMapperSupportsDimension() { + return true; + } + @Override protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "ip"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 07134fd3aa7b1..af2eaef9d3816 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -599,6 +599,11 @@ protected boolean dedupAfterFetch() { return true; } + @Override + protected boolean fieldMapperSupportsDimension() { + return true; + } + public void testDimensionInRoutingPath() throws IOException { MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("time_series_dimension", true))); IndexSettings settings = createIndexSettings( diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index a6a97b21c80ed..5ee1008c76190 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1025,8 +1025,12 @@ public final void testMinimalIsInvalidInRoutingPath() throws IOException { } } - String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - if (mapper instanceof KeywordFieldMapper || mapper instanceof NumberFieldMapper || mapper instanceof IpFieldMapper) { + protected boolean fieldMapperSupportsDimension() { + return false; + } + + private String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { + if (fieldMapperSupportsDimension()) { return "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [" diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 81848b5a50114..0e602c18929c5 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -85,6 +85,11 @@ protected Object getSampleValueForDocument() { return 123; } + @Override + protected boolean fieldMapperSupportsDimension() { + return true; + } + public void testExistsQueryDocValuesDisabled() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); From 61d37a7f961644dcad121eae3e1e0aab42cda3e3 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 5 Feb 2024 10:24:36 +0200 Subject: [PATCH 07/51] fix tests --- .../index/mapper/MapperTestCase.java | 105 ++++++++++-------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 5ee1008c76190..a2b81e21aadb0 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1037,64 +1037,71 @@ private String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { + mapper.name() + "] was not a dimension."; } - return"All fields that match routing_path must be keywords with [time_series_dimension: true] "+"or flattened fields with a list of dimensions in [time_series_dimensions] and "+"without the [script] parameter. ["+mapper.name()+"] was ["+mapper.typeName()+"]."; -} - -public record SyntheticSourceExample( - CheckedConsumer inputValue, - CheckedConsumer expectedForSyntheticSource, - CheckedConsumer expectedForBlockLoader, - CheckedConsumer mapping -) { - public SyntheticSourceExample(Object inputValue, Object result, CheckedConsumer mapping) { - this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(result), mapping); - } - - /** - * Create an example that returns different results from doc values - * than from synthetic source. - */ - public SyntheticSourceExample( - Object inputValue, - Object result, - Object blockLoaderResults, + return "All fields that match routing_path must be keywords with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + mapper.name() + + "] was [" + + mapper.typeName() + + "]."; + } + + public record SyntheticSourceExample( + CheckedConsumer inputValue, + CheckedConsumer expectedForSyntheticSource, + CheckedConsumer expectedForBlockLoader, CheckedConsumer mapping ) { - this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(blockLoaderResults), mapping); - } + public SyntheticSourceExample(Object inputValue, Object result, CheckedConsumer mapping) { + this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(result), mapping); + } - private void buildInput(XContentBuilder b) throws IOException { - b.field("field"); - inputValue.accept(b); - } + /** + * Create an example that returns different results from doc values + * than from synthetic source. + */ + public SyntheticSourceExample( + Object inputValue, + Object result, + Object blockLoaderResults, + CheckedConsumer mapping + ) { + this(b -> b.value(inputValue), b -> b.value(result), b -> b.value(blockLoaderResults), mapping); + } - private String expected() throws IOException { - XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); - expectedForSyntheticSource.accept(b); - return Strings.toString(b.endObject()); - } + private void buildInput(XContentBuilder b) throws IOException { + b.field("field"); + inputValue.accept(b); + } + + private String expected() throws IOException { + XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); + expectedForSyntheticSource.accept(b); + return Strings.toString(b.endObject()); + } - private Object expectedParsedForBlockLoader() throws IOException { - XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); - expectedForBlockLoader.accept(b); - String str = Strings.toString(b.endObject()); - return XContentHelper.convertToMap(JsonXContent.jsonXContent, str, false).get("field"); + private Object expectedParsedForBlockLoader() throws IOException { + XContentBuilder b = JsonXContent.contentBuilder().startObject().field("field"); + expectedForBlockLoader.accept(b); + String str = Strings.toString(b.endObject()); + return XContentHelper.convertToMap(JsonXContent.jsonXContent, str, false).get("field"); + } } -} -public record SyntheticSourceInvalidExample(Matcher error, CheckedConsumer mapping) {} + public record SyntheticSourceInvalidExample(Matcher error, CheckedConsumer mapping) {} -public interface SyntheticSourceSupport { - /** - * Examples that should work when source is generated from doc values. - */ - SyntheticSourceExample example(int maxValues) throws IOException; + public interface SyntheticSourceSupport { + /** + * Examples that should work when source is generated from doc values. + */ + SyntheticSourceExample example(int maxValues) throws IOException; - /** - * Examples of mappings that should be rejected when source is configured to - * be loaded from doc values. - */ - List invalidExample() throws IOException;} + /** + * Examples of mappings that should be rejected when source is configured to + * be loaded from doc values. + */ + List invalidExample() throws IOException; + } protected abstract SyntheticSourceSupport syntheticSourceSupport(boolean ignoreMalformed); From 0ddecbde83f6ce668a236644d9f03422dbaa5156 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 5 Feb 2024 14:36:16 +0200 Subject: [PATCH 08/51] refactor dimension property --- .../datastreams/TSDBIndexingIT.java | 17 +----- .../extras/TokenCountFieldMapperTests.java | 5 -- .../index/mapper/AbstractScriptFieldType.java | 2 +- .../index/mapper/DocumentMapper.java | 2 +- .../index/mapper/FieldMapper.java | 13 +++++ .../index/mapper/IpFieldMapper.java | 48 ++++++---------- .../index/mapper/KeywordFieldMapper.java | 34 +++-------- .../index/mapper/MappedFieldType.java | 57 +++++++++++++++---- .../index/mapper/NumberFieldMapper.java | 40 +++++-------- .../index/mapper/PassThroughObjectMapper.java | 10 +--- .../flattened/FlattenedFieldMapper.java | 46 ++++++--------- .../index/TimeSeriesModeTests.java | 8 +-- .../index/mapper/IpFieldMapperTests.java | 5 -- .../index/mapper/KeywordFieldMapperTests.java | 5 -- .../flattened/FlattenedFieldMapperTests.java | 4 +- .../index/mapper/MapperTestCase.java | 12 ++-- .../index/mapper/NumberFieldMapperTests.java | 5 -- .../unsignedlong/UnsignedLongFieldMapper.java | 26 +++------ 18 files changed, 142 insertions(+), 197 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java index 08b09bbc78348..772cc0f98d757 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java @@ -260,7 +260,7 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce assertThat( e.getCause().getCause().getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [metricset] was not a dimension." ) @@ -289,7 +289,7 @@ public void testInvalidTsdbTemplatesNoTimeSeriesDimensionAttribute() throws Exce } } - public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception { + public void testTsdbTemplatesNoKeywordFieldType() throws Exception { var mappingTemplate = """ { "_doc":{ @@ -315,18 +315,7 @@ public void testInvalidTsdbTemplatesNoKeywordFieldType() throws Exception { .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false)) .build() ); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet() - ); - assertThat( - e.getCause().getCause().getMessage(), - equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [metricset] was [long]." - ) - ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); } public void testInvalidTsdbTemplatesMissingSettings() throws Exception { diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java index efbf246a32f09..1636def53536b 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java @@ -62,11 +62,6 @@ protected Object getSampleValueForQuery() { return 1; } - @Override - protected boolean fieldMapperSupportsDimension() { - return true; - } - @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("index", b -> b.field("index", false)); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index 53c860530e421..62b1d2f460965 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -207,7 +207,7 @@ protected final LeafFactory leafFactory(SearchExecutionContext context) { public void validateMatchedRoutingPath(final String routingPath) { throw new IllegalArgumentException( "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " + + "must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] " + "and without the [script] parameter. [" + name() diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index f7dc09cdbb370..9b3496acfd9f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -125,7 +125,7 @@ public void validate(IndexSettings settings, boolean checkLimits) { // object type is not allowed in the routing paths if (path.equals(objectName)) { throw new IllegalArgumentException( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] " + "and without the [script] parameter. [" + objectName diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 9ed23f61bf0ea..b8731fb7e940e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1409,6 +1409,19 @@ private static boolean isDeprecatedParameter(String propName, IndexVersion index } } + public abstract static class DimensionBuilder extends Builder { + + protected boolean isDimension = false; + + public DimensionBuilder(String name) { + super(name); + } + + void setDimension() { + this.isDimension = true; + } + } + public static BiConsumer notInMultiFields(String type) { return (n, c) -> { if (c.isWithinMultiField()) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 7b1e30ef3c647..6a755d0c55f4d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -69,7 +69,7 @@ private static IpFieldMapper toType(FieldMapper in) { return (IpFieldMapper) in; } - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -166,7 +166,7 @@ protected Parameter[] getParameters() { @Override public IpFieldMapper build(MapperBuilderContext context) { - if (context.parentObjectContainsDimensions()) { + if (super.isDimension || context.parentObjectContainsDimensions()) { dimension.setValue(true); } return new IpFieldMapper( @@ -246,6 +246,21 @@ public boolean mayExistInIndex(SearchExecutionContext context) { return context.fieldExistsInIndex(name()); } + @Override + public boolean isDimension() { + return isDimension; + } + + @Override + public boolean supportsDimension() { + return true; + } + + @Override + public boolean hasScriptValues() { + return scriptValues != null; + } + private static InetAddress parse(Object value) { if (value instanceof InetAddress) { return (InetAddress) value; @@ -461,35 +476,6 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi } return terms.intersect(prefixAutomaton, searchBytes); } - - /** - * @return true if field has been marked as a dimension field - */ - public boolean isDimension() { - return isDimension; - } - - public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was not a dimension." - ); - } - if (scriptValues != null) { - throw new IllegalArgumentException( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] has a [script] parameter." - ); - } - } } private final boolean indexed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index a5a571fb82d85..74e2c4c19ebec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -137,7 +137,7 @@ private static KeywordFieldMapper toType(FieldMapper in) { return (KeywordFieldMapper) in; } - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter indexed = Parameter.indexParam(m -> toType(m).indexed, true); private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -304,7 +304,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } - if (context.parentObjectContainsDimensions()) { + if (super.isDimension || context.parentObjectContainsDimensions()) { dimension(true); } return new KeywordFieldType( @@ -811,35 +811,19 @@ public int ignoreAbove() { return ignoreAbove; } - /** - * @return true if field has been marked as a dimension field - */ @Override public boolean isDimension() { return isDimension; } @Override - public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was not a dimension." - ); - } - if (scriptValues != null) { - throw new IllegalArgumentException( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] has a [script] parameter." - ); - } + public boolean supportsDimension() { + return true; + } + + @Override + public boolean hasScriptValues() { + return scriptValues != null; } public boolean hasNormalizer() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index ea6bc2b73a208..06d191ad17a0c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -194,6 +194,20 @@ public boolean isDimension() { return false; } + /** + * @return true if field can be marked as a dimension field + */ + public boolean supportsDimension() { + return false; + } + + /** + * @return true if field has script values. + */ + public boolean hasScriptValues() { + return false; + } + /** * @return a list of dimension fields. Expected to be used by fields that have * nested fields or that, in some way, identify a collection of fields by means @@ -620,16 +634,39 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi * Validate that this field can be the target of {@link IndexMetadata#INDEX_ROUTING_PATH}. */ public void validateMatchedRoutingPath(String routingPath) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was [" - + typeName() - + "]." - ); + if (supportsDimension() == false) { + throw new IllegalArgumentException( + "All fields that match routing_path " + + "must be configured with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] was [" + + typeName() + + "]." + ); + } + + if (hasScriptValues()) { + throw new IllegalArgumentException( + "All fields that match routing_path must be configured with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] has a [script] parameter." + ); + } + + if (isDimension() == false) { + throw new IllegalArgumentException( + "All fields that match routing_path " + + "must be configured with [time_series_dimension: true] " + + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + + "without the [script] parameter. [" + + name() + + "] was not a dimension." + ); + } } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index e1ac33c132d30..da5e5e2027ab6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -89,7 +89,7 @@ private static NumberFieldMapper toType(FieldMapper in) { private static final IndexVersion MINIMUM_COMPATIBILITY_VERSION = IndexVersion.fromId(5000099); - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); @@ -267,7 +267,7 @@ protected Parameter[] getParameters() { @Override public NumberFieldMapper build(MapperBuilderContext context) { - if (context.parentObjectContainsDimensions()) { + if (super.isDimension || context.parentObjectContainsDimensions()) { dimension.setValue(true); } @@ -1745,13 +1745,21 @@ public CollapseType collapseType() { return CollapseType.NUMERIC; } - /** - * @return true if field has been marked as a dimension field - */ + @Override public boolean isDimension() { return isDimension; } + @Override + public boolean supportsDimension() { + return true; + } + + @Override + public boolean hasScriptValues() { + return scriptValues != null; + } + /** * If field is a time series metric field, returns its metric type * @return the metric type or null @@ -1759,28 +1767,6 @@ public boolean isDimension() { public MetricType getMetricType() { return metricType; } - - public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was not a dimension." - ); - } - if (scriptValues != null) { - throw new IllegalArgumentException( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] has a [script] parameter." - ); - } - } } private final NumberType type; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index daa14b1eea79c..ece16df9d6a0d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -41,14 +41,8 @@ public Builder(String name) { @Override public PassThroughObjectMapper.Builder add(Mapper.Builder builder) { - if (timeSeriesDimensionSubFields.value()) { - if (builder instanceof KeywordFieldMapper.Builder keywordBuilder) { - keywordBuilder.dimension(true); - } else if (builder instanceof NumberFieldMapper.Builder numberBuilder) { - numberBuilder.dimension(true); - } else if (builder instanceof IpFieldMapper.Builder ipBuilder) { - ipBuilder.dimension(true); - } + if (timeSeriesDimensionSubFields.value() && builder instanceof FieldMapper.DimensionBuilder dimensionBuilder) { + dimensionBuilder.setDimension(); } super.add(builder); return this; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index c15adfb3be116..575b9e8e17ef5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -231,6 +231,16 @@ public static final class KeyedFlattenedFieldType extends StringFieldType { private final String rootName; private final boolean isDimension; + @Override + public boolean isDimension() { + return isDimension; + } + + @Override + public boolean supportsDimension() { + return true; + } + KeyedFlattenedFieldType( String rootName, boolean indexed, @@ -280,24 +290,6 @@ public Query existsQuery(SearchExecutionContext context) { return new PrefixQuery(term); } - @Override - public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + this.rootName - + "." - + this.key - + "] was [" - + typeName() - + "]." - ); - } - } - @Override public Query rangeQuery( Object lowerTerm, @@ -730,6 +722,11 @@ public boolean isDimension() { return isDimension; } + @Override + public boolean supportsDimension() { + return true; + } + @Override public List dimensions() { return this.dimensions; @@ -737,17 +734,8 @@ public List dimensions() { @Override public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension && this.dimensions.contains(routingPath) == false) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was [" - + typeName() - + "]." - ); + if (this.dimensions.contains(routingPath) == false) { + super.validateMatchedRoutingPath(routingPath); } } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index 847bb1dfbfb0b..6118a84814462 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -174,7 +174,7 @@ public void testRoutingPathEqualsObjectNameError() { assertThat( e.getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [dim.o] was [object]." ) @@ -192,7 +192,7 @@ public void testRoutingPathMatchesNonDimensionKeyword() { assertThat( e.getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [dim.non_dim] was not a dimension." ) @@ -221,7 +221,7 @@ public void testRoutingPathMatchesScriptedKeyword() { assertThat( e.getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [dim.kwd] has a [script] parameter." ) @@ -237,7 +237,7 @@ public void testRoutingPathMatchesRuntimeKeyword() { assertThat( e.getMessage(), equalTo( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [dim.kwd] was a runtime [keyword]." ) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index f06bc457ec9bb..ba9c2e6c4a299 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -43,11 +43,6 @@ protected Object getSampleValueForDocument() { return "::1"; } - @Override - protected boolean fieldMapperSupportsDimension() { - return true; - } - @Override protected void minimalMapping(XContentBuilder b) throws IOException { b.field("type", "ip"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index af2eaef9d3816..07134fd3aa7b1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -599,11 +599,6 @@ protected boolean dedupAfterFetch() { return true; } - @Override - protected boolean fieldMapperSupportsDimension() { - return true; - } - public void testDimensionInRoutingPath() throws IOException { MapperService mapper = createMapperService(fieldMapping(b -> b.field("type", "keyword").field("time_series_dimension", true))); IndexSettings settings = createIndexSettings( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index e4ea78f3b7a0e..4f23c86f53cca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -467,9 +467,9 @@ public void testMissingDimensionInRoutingPath() throws IOException { ); Exception ex = expectThrows(IllegalArgumentException.class, () -> mapper.documentMapper().validate(settings, false)); assertEquals( - "All fields that match routing_path must be keywords with [time_series_dimension: true] " + "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [field.key3] was [flattened].", + + "without the [script] parameter. [field._keyed] was not a dimension.", ex.getMessage() ); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index a2b81e21aadb0..37439b691f6c6 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1025,19 +1025,17 @@ public final void testMinimalIsInvalidInRoutingPath() throws IOException { } } - protected boolean fieldMapperSupportsDimension() { - return false; - } - private String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - if (fieldMapperSupportsDimension()) { - return "All fields that match routing_path must be keywords with [time_series_dimension: true] " + if (mapper instanceof FieldMapper fieldMapper + && fieldMapper.fieldType().supportsDimension() + && fieldMapper.fieldType().isDimension() == false) { + return "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [" + mapper.name() + "] was not a dimension."; } - return "All fields that match routing_path must be keywords with [time_series_dimension: true] " + return "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [" + mapper.name() diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 0e602c18929c5..81848b5a50114 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -85,11 +85,6 @@ protected Object getSampleValueForDocument() { return 123; } - @Override - protected boolean fieldMapperSupportsDimension() { - return true; - } - public void testExistsQueryDocValuesDisabled() throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> { minimalMapping(b); diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index ded6e135ad0f1..8a6ff8d622b50 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -76,7 +76,7 @@ private static UnsignedLongFieldMapper toType(FieldMapper in) { return (UnsignedLongFieldMapper) in; } - public static final class Builder extends FieldMapper.Builder { + public static final class Builder extends FieldMapper.DimensionBuilder { private final Parameter indexed; private final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); private final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); @@ -195,7 +195,7 @@ Number parsedNullValue() { @Override public UnsignedLongFieldMapper build(MapperBuilderContext context) { - if (context.parentObjectContainsDimensions()) { + if (super.isDimension || context.parentObjectContainsDimensions()) { dimension.setValue(true); } UnsignedLongFieldType fieldType = new UnsignedLongFieldType( @@ -539,13 +539,16 @@ static Long parseUpperRangeTerm(Object value, boolean include) { return longValue; } - /** - * @return true if field has been marked as a dimension field - */ + @Override public boolean isDimension() { return isDimension; } + @Override + public boolean supportsDimension() { + return true; + } + /** * If field is a time series metric field, returns its metric type * @return the metric type or null @@ -553,19 +556,6 @@ public boolean isDimension() { public MetricType getMetricType() { return metricType; } - - public void validateMatchedRoutingPath(final String routingPath) { - if (false == isDimension) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be keywords with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was not a dimension." - ); - } - } } private final boolean indexed; From fa065b0b8aae903b1373c5e49fb447dd82b04098 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 5 Feb 2024 15:23:25 +0200 Subject: [PATCH 09/51] fix yaml --- .../resources/rest-api-spec/test/tsdb/140_routing_path.yml | 2 +- .../resources/rest-api-spec/test/tsdb/20_mapping.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 5813445326ef6..d5471ffd87159 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -94,7 +94,7 @@ missing dimension on routing path field: reason: error message changed in 8.8.0 - do: - catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[tag\] was not a dimension./' + catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[tag\] was not a dimension./' indices.create: index: test body: diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 7edae8f264c76..120d85806c5bc 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -125,7 +125,7 @@ exact match object type: reason: routing_path error message updated in 8.8.0 - do: - catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./' + catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./' indices.create: index: tsdb_index body: @@ -158,7 +158,7 @@ non keyword matches routing_path: reason: routing_path error message updated in 8.8.0 - do: - catch: '/All fields that match routing_path must be keywords with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./' + catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./' indices.create: index: test_index body: @@ -273,7 +273,7 @@ runtime field matching routing path: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim_kw": "dim", "dim": {"foo": "a"}}' - - match: {items.0.index.error.reason: "All fields that match routing_path must be keywords with [time_series_dimension: true] or flattened fields with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [dim.foo] was a runtime [keyword]."} + - match: {items.0.index.error.reason: "All fields that match routing_path must be configured with [time_series_dimension: true] or flattened fields with a list of dimensions in [time_series_dimensions] and without the [script] parameter. [dim.foo] was a runtime [keyword]."} --- "dynamic: false matches routing_path": From c712893858f0d20a5291b2ea3fba70038a6f42dc Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 5 Feb 2024 15:59:24 +0200 Subject: [PATCH 10/51] add numeric to routing builder --- .../cluster/routing/IndexRouting.java | 5 +---- .../index/mapper/TimeSeriesIdFieldMapper.java | 6 ++++++ .../index/mapper/TsidExtractingIdFieldMapper.java | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 1ed9d759c4ca8..543d227aa378a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -334,16 +334,13 @@ private void extractItem(String path, XContentParser source) throws IOException source.nextToken(); break; case VALUE_STRING: + case VALUE_NUMBER: hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); source.nextToken(); break; case VALUE_NULL: source.nextToken(); break; - case VALUE_NUMBER: // allow parsing numbers assuming routing fields are always keyword fields - hashes.add(new NameAndHash(new BytesRef(path), hash(new BytesRef(source.text())))); - source.nextToken(); - break; default: throw new ParsingException( source.getTokenLocation(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index d7b8a9a49970e..949cc01e354b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -305,6 +305,9 @@ public DocumentDimensions addLong(String fieldName, long value) { out.write((byte) 'l'); out.writeLong(value); add(fieldName, out.bytes()); + if (routingBuilder != null) { + routingBuilder.addMatching(fieldName, new BytesRef(Long.toString(value).getBytes())); + } } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } @@ -323,6 +326,9 @@ public DocumentDimensions addUnsignedLong(String fieldName, long value) { out.writeLong(value); } add(fieldName, out.bytes()); + if (routingBuilder != null) { + routingBuilder.addMatching(fieldName, new BytesRef(Long.toString(value).getBytes())); + } return this; } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index ad8bc71d5f8a0..27e281ed7fb52 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -116,6 +116,20 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra long timestamp = timestampFields.get(0).numericValue().longValue(); byte[] suffix = new byte[16]; String id = createId(context.hasDynamicMappers() == false, routingBuilder, tsid, timestamp, suffix); + /* + * Make sure that _id from extracting the tsid matches that _id + * from extracting the _source. This should be true for all valid + * documents with valid mappings. *But* some invalid mappings + * will not parse the field but be rejected later by the dynamic + * mappings machinery. So if there are any dynamic mappings + * at all we just skip the assertion because we can't be sure + * it always must pass. + */ + IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); + assert context.getDynamicMappers().isEmpty() == false + || context.getDynamicRuntimeFields().isEmpty() == false + || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); + if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { throw new IllegalArgumentException( String.format( From 03784f7d520764dfc187296ae53791d16f514870 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 5 Feb 2024 16:09:59 +0200 Subject: [PATCH 11/51] fix violation --- .../elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 949cc01e354b2..6113deee1c787 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.net.InetAddress; +import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.util.Base64; import java.util.Collections; @@ -306,7 +307,7 @@ public DocumentDimensions addLong(String fieldName, long value) { out.writeLong(value); add(fieldName, out.bytes()); if (routingBuilder != null) { - routingBuilder.addMatching(fieldName, new BytesRef(Long.toString(value).getBytes())); + routingBuilder.addMatching(fieldName, new BytesRef(Long.toString(value).getBytes(StandardCharsets.UTF_8))); } } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); @@ -327,7 +328,7 @@ public DocumentDimensions addUnsignedLong(String fieldName, long value) { } add(fieldName, out.bytes()); if (routingBuilder != null) { - routingBuilder.addMatching(fieldName, new BytesRef(Long.toString(value).getBytes())); + routingBuilder.addMatching(fieldName, new BytesRef(Long.toString(value).getBytes(StandardCharsets.UTF_8))); } return this; } catch (IOException e) { From 5c23358dc069bce00c0de63fc2cbcebb2e747288 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 5 Feb 2024 16:17:29 +0200 Subject: [PATCH 12/51] fix yaml versions --- .../rest-api-spec/test/tsdb/140_routing_path.yml | 4 ++-- .../resources/rest-api-spec/test/tsdb/20_mapping.yml | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index d5471ffd87159..53a1821ee8351 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -90,8 +90,8 @@ missing routing path field: --- missing dimension on routing path field: - skip: - version: " - 8.7.99" - reason: error message changed in 8.8.0 + version: " - 8.12.99" + reason: error message changed in 8.13.0 - do: catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[tag\] was not a dimension./' diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 120d85806c5bc..9aa971e521de8 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -121,8 +121,8 @@ top level wildcard dim object: --- exact match object type: - skip: - version: " - 8.7.99" - reason: routing_path error message updated in 8.8.0 + version: " - 8.12.99" + reason: routing_path error message updated in 8.13.0 - do: catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./' @@ -154,8 +154,8 @@ exact match object type: --- non keyword matches routing_path: - skip: - version: " - 8.7.99" - reason: routing_path error message updated in 8.8.0 + version: " - 8.12.99" + reason: routing_path error message updated in 8.13.0 - do: catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./' @@ -241,8 +241,8 @@ runtime field matching routing path: --- "dynamic: runtime matches routing_path": - skip: - version: " - 8.7.99" - reason: routing_path error message updated in 8.8.0 + version: " - 8.12.99" + reason: routing_path error message updated in 8.13.0 - do: indices.create: From d73c472d54af9830b9ce79b0d7dd77ed03d8bd95 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 6 Feb 2024 09:56:19 +0200 Subject: [PATCH 13/51] handle all numeric values in dimension fields --- .../test/data_stream/150_tsdb.yml | 33 +++++++++++++++---- .../index/mapper/DocumentDimensions.java | 8 +++++ .../index/mapper/NumberFieldMapper.java | 14 ++++---- .../index/mapper/TimeSeriesIdFieldMapper.java | 17 ++++++++++ .../index/mapper/NumberFieldMapperTests.java | 8 ++--- 5 files changed, 60 insertions(+), 20 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 437bae15cdfc9..54ad08684399a 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -230,13 +230,13 @@ dynamic templates: refresh: true body: - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "attributes.another.dim": 10 }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "attributes.int.dim": 10, "attributes.double.dim": 10.5 }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "attributes.another.dim": 10 }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "attributes.int.dim": 10, "attributes.double.dim": 10.5 }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "attributes.another.dim": 20 }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "attributes.int.dim": 20, "attributes.double.dim": 20.5 }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "attributes.another.dim": 20 }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "attributes.int.dim": 20, "attributes.double.dim": 20.5 }' - do: search: @@ -262,7 +262,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur7qNQVbF9nXxeO_bMXMxPMOBKO-Urpf2SU" } + - match: { aggregations.filterA.tsids.buckets.0.key: "LPWTjzBxe1mblxCxXB3bXoUX2dfFHLp75Oo1BVv9pT9Uy36DR_ifC8rajKv3" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -274,14 +274,33 @@ dynamic templates: filterA: filter: term: - another.dim: 10 + int.dim: 10 aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur7qNQVbF9nXxeO_bMXMxPMOBKO-Urpf2SU" } + - match: { aggregations.filterA.tsids.buckets.0.key: "LPWTjzBxe1mblxCxXB3bXoUX2dfFHLp75Oo1BVv9pT9Uy36DR_ifC8rajKv3" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + double.dim: 10.5 + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "LPWTjzBxe1mblxCxXB3bXoUX2dfFHLp75Oo1BVv9pT9Uy36DR_ifC8rajKv3" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java index 3cf90f2385525..043db8a33a4be 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java @@ -42,6 +42,8 @@ default DocumentDimensions addString(String fieldName, String value) { DocumentDimensions addLong(String fieldName, long value); + DocumentDimensions addDouble(String fieldName, double value); + DocumentDimensions addUnsignedLong(String fieldName, long value); DocumentDimensions validate(IndexSettings settings); @@ -77,6 +79,12 @@ public DocumentDimensions addLong(String fieldName, long value) { return this; } + @Override + public DocumentDimensions addDouble(String fieldName, double value) { + add(fieldName); + return this; + } + @Override public DocumentDimensions addUnsignedLong(String fieldName, long value) { add(fieldName); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index da5e5e2027ab6..6f16ed4e1f4d4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -70,7 +70,6 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -183,11 +182,6 @@ public Builder( } }); this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).dimension).addValidator(v -> { - if (v && EnumSet.of(NumberType.INTEGER, NumberType.LONG, NumberType.BYTE, NumberType.SHORT).contains(type) == false) { - throw new IllegalArgumentException( - "Parameter [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] cannot be set to numeric type [" + type.name + "]" - ); - } if (v && (indexed.getValue() == false || hasDocValues.getValue() == false)) { throw new IllegalArgumentException( "Field [" @@ -1891,7 +1885,13 @@ public Number value(XContentParser parser) throws IllegalArgumentException, IOEx */ public void indexValue(DocumentParserContext context, Number numericValue) { if (dimension && numericValue != null) { - context.getDimensions().addLong(fieldType().name(), numericValue.longValue()).validate(context.indexSettings()); + DocumentDimensions dimensions = switch (type) { + case DOUBLE -> context.getDimensions().addDouble(fieldType().name(), numericValue.doubleValue()); + case FLOAT, HALF_FLOAT -> context.getDimensions().addDouble(fieldType().name(), numericValue.floatValue()); + case LONG -> context.getDimensions().addLong(fieldType().name(), numericValue.longValue()); + case INTEGER, SHORT, BYTE -> context.getDimensions().addLong(fieldType().name(), numericValue.intValue()); + }; + dimensions.validate(context.indexSettings()); } fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 6113deee1c787..f02aba1bff85f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -336,6 +336,21 @@ public DocumentDimensions addUnsignedLong(String fieldName, long value) { } } + @Override + public DocumentDimensions addDouble(String fieldName, double value) { + try (BytesStreamOutput out = new BytesStreamOutput()) { + out.write((byte) 'd'); + out.writeDouble(value); + add(fieldName, out.bytes()); + if (routingBuilder != null) { + routingBuilder.addMatching(fieldName, new BytesRef(Double.toString(value).getBytes(StandardCharsets.UTF_8))); + } + } catch (IOException e) { + throw new IllegalArgumentException("Dimension field cannot be serialized.", e); + } + return this; + } + @Override public DocumentDimensions validate(final IndexSettings settings) { if (settings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ID_HASHING) @@ -407,6 +422,8 @@ public static Map decodeTsidAsMap(StreamInput in) { Object ul = DocValueFormat.UNSIGNED_LONG_SHIFTED.format(in.readLong()); result.put(name, ul); } + case (byte) 'd' -> // parse a double + result.put(name, in.readDouble()); default -> throw new IllegalArgumentException("Cannot parse [" + name + "]: Unknown type [" + type + "]"); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index 81848b5a50114..c60a913a63b33 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -253,12 +253,8 @@ public void testDimension() throws IOException { // dimension = false is allowed assertDimension(false, NumberFieldMapper.NumberFieldType::isDimension); - // dimension = true is not allowed - Exception e = expectThrows(MapperParsingException.class, () -> createDocumentMapper(fieldMapping(b -> { - minimalMapping(b); - b.field("time_series_dimension", true); - }))); - assertThat(e.getCause().getMessage(), containsString("Parameter [time_series_dimension] cannot be set")); + // dimension = true is allowed + assertDimension(true, NumberFieldMapper.NumberFieldType::isDimension); } public void testMetricType() throws IOException { From e2887a0fcb2e6981856c71e2d37a560a5be05fcd Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 13 Feb 2024 13:24:40 +0200 Subject: [PATCH 14/51] use shardId in TSDB doc id --- .../timeseries/TimeSeriesAggregator.java | 2 +- .../timeseries/InternalTimeSeriesTests.java | 2 +- .../timeseries/TimeSeriesAggregatorTests.java | 2 +- .../TSDBPassthroughIndexingIT.java | 20 ++++- .../test/data_stream/150_tsdb.yml | 61 +++++++++----- .../cluster/routing/IndexRouting.java | 9 +-- .../org/elasticsearch/index/IndexMode.java | 4 +- .../index/engine/TranslogDirectoryReader.java | 3 +- .../index/mapper/DocumentDimensions.java | 8 -- .../index/mapper/DocumentMapper.java | 6 +- .../index/mapper/DocumentParser.java | 10 ++- .../index/mapper/DocumentParserContext.java | 19 ++++- .../elasticsearch/index/mapper/IdLoader.java | 39 ++------- .../index/mapper/NumberFieldMapper.java | 8 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 37 +-------- .../mapper/TsidExtractingIdFieldMapper.java | 49 +++--------- .../index/query/SearchExecutionContext.java | 2 +- .../elasticsearch/index/shard/IndexShard.java | 8 +- .../index/termvectors/TermVectorsService.java | 2 +- .../search/DefaultSearchContext.java | 23 +----- .../elasticsearch/search/DocValueFormat.java | 2 +- .../common/lucene/uid/VersionsTests.java | 14 +--- .../index/mapper/BlockSourceReaderTests.java | 2 +- .../index/mapper/IdLoaderTests.java | 59 ++++++-------- .../TsidExtractingIdFieldMapperTests.java | 80 ++++++++++--------- .../recovery/RecoverySourceHandlerTests.java | 6 +- .../search/DocValueFormatTests.java | 2 +- .../index/engine/TranslogHandler.java | 3 +- .../mapper/TestDocumentParserContext.java | 3 +- .../rate/TimeSeriesRateAggregatorTests.java | 2 +- .../ValuesSourceReaderOperatorTests.java | 3 +- .../aggregations/GeoLineAggregatorTests.java | 2 +- 32 files changed, 204 insertions(+), 288 deletions(-) diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java index 9cd7f7a86e532..d29a964b526f9 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java @@ -157,7 +157,7 @@ public void collect(int doc, long bucket) throws IOException { return; } - TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); for (TsidConsumer consumer : dimensionConsumers.values()) { consumer.accept(doc, tsidBuilder); } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index a77ea04a2d8a1..c6cd161b879da 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -41,7 +41,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a List> keys = randomKeys(bucketKeys(randomIntBetween(1, 4)), numberOfBuckets); for (int j = 0; j < numberOfBuckets; j++) { long docCount = randomLongBetween(0, Long.MAX_VALUE / (20L * numberOfBuckets)); - var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); for (var entry : keys.get(j).entrySet()) { builder.addString(entry.getKey(), (String) entry.getValue()); } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 54d1c14931d92..8c10d55b9acd4 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -92,7 +92,7 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens final List fields = new ArrayList<>(); fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); - final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); + final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(); for (int i = 0; i < dimensions.length; i += 2) { if (dimensions[i + 1] instanceof Number n) { builder.addLong(dimensions[i].toString(), n.longValue()); diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index b58aa201d077f..dda114e5cd8f0 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -46,6 +46,17 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase { public static final String MAPPING_TEMPLATE = """ { "_doc":{ + "dynamic_templates": [ + { + "strings_as_ip": { + "match_mapping_type": "string", + "match": "*ip", + "mapping": { + "type": "ip" + } + } + } + ], "properties": { "@timestamp" : { "type": "date" @@ -80,6 +91,7 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase { "@timestamp": "$time", "attributes": { "metricset": "pod", + "number": $number, "pod": { "name": "$name", "uid": "$uid", @@ -134,6 +146,7 @@ public void testIndexingGettingAndSearching() throws Exception { DOC.replace("$time", formatInstant(time)) .replace("$uid", randomUUID()) .replace("$name", randomAlphaOfLength(4)) + .replace("$number", Long.toString(randomLong())) .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))), XContentType.JSON ); @@ -165,7 +178,8 @@ public void testIndexingGettingAndSearching() throws Exception { ); @SuppressWarnings("unchecked") var attributes = (Map>) ObjectPath.eval("properties.attributes.properties", mapping); - assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true)); + assertMap(attributes.get("number"), matchesMap().entry("type", "long").entry("time_series_dimension", true)); + assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "ip").entry("time_series_dimension", true)); assertMap(attributes.get("pod.uid"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true)); assertMap(attributes.get("pod.name"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true)); // alias field mappers: @@ -173,6 +187,10 @@ public void testIndexingGettingAndSearching() throws Exception { ObjectPath.eval("properties.metricset", mapping), matchesMap().entry("type", "alias").entry("path", "attributes.metricset") ); + assertMap( + ObjectPath.eval("properties.number", mapping), + matchesMap().entry("type", "alias").entry("path", "attributes.number") + ); assertMap( ObjectPath.eval("properties.pod.properties", mapping), matchesMap().extraOk().entry("name", matchesMap().entry("type", "alias").entry("path", "attributes.pod.name")) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index d96d1b47cc6af..e45407e4c2129 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -230,13 +230,13 @@ dynamic templates: refresh: true body: - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes.dim": "A", "attributes.another.dim": "C" }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z", "data": "10", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "attributes.dim": "A", "attributes.another.dim": "C" }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z", "data": "20", "attributes.dim1": "A", "attributes.dim2": "1", "attributes.another.dim1": "C", "attributes.another.dim2": "10.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "attributes.dim": "B", "attributes.another.dim": "D" }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z", "data": "30", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "attributes.dim": "B", "attributes.another.dim": "D" }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z", "data": "40", "attributes.dim1": "B", "attributes.dim2": "2", "attributes.another.dim1": "D", "attributes.another.dim2": "20.5" }' - do: search: @@ -255,14 +255,14 @@ dynamic templates: filterA: filter: term: - dim: A + dim1: A aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "KPLatIF-tbSjI8l9LcbTur4gYpiKF9nXxRUMHnJBiqqP58deEIun8H8" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -274,14 +274,14 @@ dynamic templates: filterA: filter: term: - another.dim: C + dim2: 1 aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -300,7 +300,7 @@ dynamic templates: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -312,14 +312,14 @@ dynamic templates: filterA: filter: term: - another.dim2: 10 + another.dim2: 10.5 aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiK8yYWLhfZ18WLDvTuBX1YJX1Ll7UMNJqYNES5Eg" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MD2HE8yse1ZklY-p0-bRcC8gYpiKqVppKhfZ18WLDvTuNPo7EnyZdkhvafL006Xf2Q" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- @@ -463,13 +463,13 @@ dynamic templates with nesting: refresh: true body: - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim": "A", "resource.attributes.another.dim": "C", "attributes.more.dim": "E" }' + - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim": "A", "resource.attributes.another.dim": "C", "attributes.more.dim": "E" }' + - '{ "@timestamp": "2023-09-01T13:03:09.138Z","data": "20", "resource.attributes.dim1": "A", "resource.attributes.another.dim1": "1", "attributes.dim2": "C", "attributes.another.dim2": "10.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim": "B", "resource.attributes.another.dim": "D", "attributes.more.dim": "F" }' + - '{ "@timestamp": "2023-09-01T13:03:10.138Z","data": "30", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5" }' - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }' - - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim": "B", "resource.attributes.another.dim": "D", "attributes.more.dim": "F" }' + - '{ "@timestamp": "2023-09-01T13:03:10.238Z","data": "40", "resource.attributes.dim1": "B", "resource.attributes.another.dim1": "2", "attributes.dim2": "D", "attributes.another.dim2": "20.5" }' - do: search: @@ -488,14 +488,14 @@ dynamic templates with nesting: filterA: filter: term: - dim: A + dim1: A aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -507,14 +507,14 @@ dynamic templates with nesting: filterA: filter: term: - another.dim: C + dim2: C aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "LEjiJ4ATCXWlzeFvhGQ9lYlnP-nRIGKYihfZ18WoJ94t9a8OpbsCdwZALomb" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } - do: @@ -526,14 +526,33 @@ dynamic templates with nesting: filterA: filter: term: - another.dim2: 10 + another.dim1: 1 aggs: tsids: terms: field: _tsid - length: { aggregations.filterA.tsids.buckets: 1 } - - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK7zJhYuIGKYiosO9O4X2dfFtp-JEbk39FSSMEq_vwX7uw" } + - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" } + - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } + + - do: + search: + index: k9s + body: + size: 0 + aggs: + filterA: + filter: + term: + another.dim2: 10.5 + aggs: + tsids: + terms: + field: _tsid + + - length: { aggregations.filterA.tsids.buckets: 1 } + - match: { aggregations.filterA.tsids.buckets.0.key: "MK0AtuFZowY4QPzoYEAZNK6pWmkqIGKYiosO9O4X2dfFL8p_4TfsFAUUYYv9EqSmEQ" } - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 } --- diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 543d227aa378a..7bf6149c6cc23 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -59,7 +59,7 @@ public static IndexRouting fromIndexMetadata(IndexMetadata metadata) { } protected final String indexName; - private final int routingNumShards; + protected final int routingNumShards; private final int routingFactor; private IndexRouting(IndexMetadata metadata) { @@ -248,10 +248,6 @@ public static class ExtractFromSource extends IndexRouting { this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); } - public boolean matchesField(String fieldName) { - return isRoutingPath.test(fieldName); - } - @Override public void process(IndexRequest indexRequest) {} @@ -408,7 +404,8 @@ private int idToHash(String id) { if (idBytes.length < 4) { throw new ResourceNotFoundException("invalid id [{}] for index [{}] in time series mode", id, indexName); } - return hashToShardId(ByteUtils.readIntLE(idBytes, 0)); + int shardId = ByteUtils.readIntLE(idBytes, 0); + return Math.floorMod(shardId, routingNumShards); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 05afc14e0f0cd..67bd322c87c53 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -10,7 +10,6 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -197,8 +196,7 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { @Override public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { - IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); - return new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); + return new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index 6e1c01c886145..d7c91c61987ba 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -258,7 +258,8 @@ private LeafReader createInMemoryLeafReader() { Map.of(), false ), - mappingLookup + mappingLookup, + shardId.id() ); parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java index 043db8a33a4be..3cf90f2385525 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentDimensions.java @@ -42,8 +42,6 @@ default DocumentDimensions addString(String fieldName, String value) { DocumentDimensions addLong(String fieldName, long value); - DocumentDimensions addDouble(String fieldName, double value); - DocumentDimensions addUnsignedLong(String fieldName, long value); DocumentDimensions validate(IndexSettings settings); @@ -79,12 +77,6 @@ public DocumentDimensions addLong(String fieldName, long value) { return this; } - @Override - public DocumentDimensions addDouble(String fieldName, double value) { - add(fieldName); - return this; - } - @Override public DocumentDimensions addUnsignedLong(String fieldName, long value) { add(fieldName); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index 9b3496acfd9f3..c5b26850b9753 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -89,7 +89,11 @@ public MappingLookup mappers() { } public ParsedDocument parse(SourceToParse source) throws DocumentParsingException { - return documentParser.parseDocument(source, mappingLookup); + return parse(source, 0); + } + + public ParsedDocument parse(SourceToParse source, int shardId) throws DocumentParsingException { + return documentParser.parseDocument(source, mappingLookup, shardId); } public void validate(IndexSettings settings, boolean checkLimits) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index fe6b0b2051dc9..8c5e8037d6fde 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -74,7 +74,7 @@ public final class DocumentParser { * @return the parsed document * @throws DocumentParsingException whenever there's a problem parsing the document */ - public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingLookup) throws DocumentParsingException { + public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingLookup, int shardId) throws DocumentParsingException { if (source.source() != null && source.source().length() == 0) { throw new DocumentParsingException(new XContentLocation(0, 0), "failed to parse, document is empty"); } @@ -90,7 +90,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL XContentHelper.createParser(parserConfiguration, source.source(), xContentType) ) ) { - context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); + context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser, shardId); validateStart(context.parser()); MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers(); internalParseDocument(metadataFieldsMappers, context); @@ -853,14 +853,16 @@ private static class RootDocumentParserContext extends DocumentParserContext { MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, - XContentParser parser + XContentParser parser, + int shardId ) throws IOException { super( mappingLookup, mappingParserContext, source, mappingLookup.getMapping().getRoot(), - ObjectMapper.Dynamic.getRootDynamic(mappingLookup) + ObjectMapper.Dynamic.getRootDynamic(mappingLookup), + shardId ); if (mappingLookup.getMapping().getRoot().subobjects()) { this.parser = DotExpandingXContentParser.expandDots(parser, this.path); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 66c5de61bcd92..87eabe0f8fabf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -115,6 +115,8 @@ public int get() { private final Set fieldsAppliedFromTemplates; private final Set copyToFields; + private final int shardId; + private DocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, @@ -131,7 +133,8 @@ private DocumentParserContext( ObjectMapper.Dynamic dynamic, Set fieldsAppliedFromTemplates, Set copyToFields, - DynamicMapperSize dynamicMapperSize + DynamicMapperSize dynamicMapperSize, + final int shardId ) { this.mappingLookup = mappingLookup; this.mappingParserContext = mappingParserContext; @@ -149,6 +152,7 @@ private DocumentParserContext( this.fieldsAppliedFromTemplates = fieldsAppliedFromTemplates; this.copyToFields = copyToFields; this.dynamicMappersSize = dynamicMapperSize; + this.shardId = shardId; } private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, DocumentParserContext in) { @@ -168,7 +172,8 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, dynamic, in.fieldsAppliedFromTemplates, in.copyToFields, - in.dynamicMappersSize + in.dynamicMappersSize, + in.shardId ); } @@ -177,7 +182,8 @@ protected DocumentParserContext( MappingParserContext mappingParserContext, SourceToParse source, ObjectMapper parent, - ObjectMapper.Dynamic dynamic + ObjectMapper.Dynamic dynamic, + int shardId ) { this( mappingLookup, @@ -195,7 +201,8 @@ protected DocumentParserContext( dynamic, new HashSet<>(), new HashSet<>(), - new DynamicMapperSize() + new DynamicMapperSize(), + shardId ); } @@ -231,6 +238,10 @@ public final SourceToParse sourceToParse() { return this.sourceToParse; } + public int getShardId() { + return shardId; + } + /** * Add the given {@code field} to the set of ignored fields. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index c965a77f1b5bf..9073fe7ccdafc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -12,14 +12,11 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import java.io.IOException; -import java.util.List; /** * Responsible for loading the _id from stored fields or for TSDB synthesizing the _id from the routing, _tsid and @timestamp fields. @@ -36,8 +33,8 @@ static IdLoader fromLeafStoredFieldLoader() { /** * @return returns an {@link IdLoader} instance that syn synthesizes _id from routing, _tsid and @timestamp fields. */ - static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { - return new TsIdLoader(indexRouting, routingPaths); + static IdLoader createTsIdLoader(int shardId) { + return new TsIdLoader(shardId); } Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException; @@ -57,35 +54,13 @@ sealed interface Leaf permits StoredLeaf, TsIdLeaf { final class TsIdLoader implements IdLoader { - private final IndexRouting.ExtractFromSource indexRouting; - private final List routingPaths; + final int shardId; - TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { - this.routingPaths = routingPaths; - this.indexRouting = indexRouting; + TsIdLoader(int shardId) { + this.shardId = shardId; } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { - IndexRouting.ExtractFromSource.Builder[] builders = new IndexRouting.ExtractFromSource.Builder[docIdsInLeaf.length]; - for (int i = 0; i < builders.length; i++) { - builders[i] = indexRouting.builder(); - } - - for (String routingField : routingPaths) { - // Routing field must always be keyword fields, so it is ok to use SortedSetDocValues directly here. - SortedSetDocValues dv = DocValues.getSortedSet(reader, routingField); - for (int i = 0; i < docIdsInLeaf.length; i++) { - int docId = docIdsInLeaf[i]; - var builder = builders[i]; - if (dv.advanceExact(docId)) { - for (int j = 0; j < dv.docValueCount(); j++) { - BytesRef routingValue = dv.lookupOrd(dv.nextOrd()); - builder.addMatching(routingField, routingValue); - } - } - } - } - String[] ids = new String[docIdsInLeaf.length]; // Each document always has exactly one tsid and one timestamp: SortedDocValues tsIdDocValues = DocValues.getSorted(reader, TimeSeriesIdFieldMapper.NAME); @@ -100,9 +75,7 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] assert found; assert timestampDocValues.docValueCount() == 1; long timestamp = timestampDocValues.nextValue(); - - var routingBuilder = builders[i]; - ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); + ids[i] = TsidExtractingIdFieldMapper.createId(shardId, tsid, timestamp); } return new TsIdLeaf(docIdsInLeaf, ids); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6f16ed4e1f4d4..cbd052ad4a1d7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1885,13 +1885,7 @@ public Number value(XContentParser parser) throws IllegalArgumentException, IOEx */ public void indexValue(DocumentParserContext context, Number numericValue) { if (dimension && numericValue != null) { - DocumentDimensions dimensions = switch (type) { - case DOUBLE -> context.getDimensions().addDouble(fieldType().name(), numericValue.doubleValue()); - case FLOAT, HALF_FLOAT -> context.getDimensions().addDouble(fieldType().name(), numericValue.floatValue()); - case LONG -> context.getDimensions().addLong(fieldType().name(), numericValue.longValue()); - case INTEGER, SHORT, BYTE -> context.getDimensions().addLong(fieldType().name(), numericValue.intValue()); - }; - dimensions.validate(context.indexSettings()); + context.getDimensions().addLong(fieldType().name(), numericValue.longValue()).validate(context.indexSettings()); } fieldType().type.addFields(context.doc(), fieldType().name(), numericValue, indexed, hasDocValues, stored); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 6da546a4fd383..22e9b99d8877a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -12,7 +12,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.Murmur3Hasher; @@ -21,7 +20,6 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -38,7 +36,6 @@ import java.io.IOException; import java.net.InetAddress; -import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.util.Base64; import java.util.Collections; @@ -140,7 +137,7 @@ public void postParse(DocumentParserContext context) throws IOException { ? timeSeriesIdBuilder.buildLegacyTsid().toBytesRef() : timeSeriesIdBuilder.buildTsidHash().toBytesRef(); context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); - TsidExtractingIdFieldMapper.createField(context, timeSeriesIdBuilder.routingBuilder, timeSeriesId); + TsidExtractingIdFieldMapper.createField(context, timeSeriesId); } private IndexVersion getIndexVersionCreated(final DocumentParserContext context) { @@ -184,15 +181,10 @@ private record Dimension(BytesRef name, BytesReference value) {} * to build the _tsid field for the document. */ private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); + /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ - @Nullable - private final IndexRouting.ExtractFromSource.Builder routingBuilder; - - public TimeSeriesIdBuilder(@Nullable IndexRouting.ExtractFromSource.Builder routingBuilder) { - this.routingBuilder = routingBuilder; - } public BytesReference buildLegacyTsid() throws IOException { if (dimensions.isEmpty()) { @@ -285,10 +277,6 @@ public DocumentDimensions addString(String fieldName, BytesRef utf8Value) { */ out.writeBytesRef(utf8Value); add(fieldName, out.bytes()); - - if (routingBuilder != null) { - routingBuilder.addMatching(fieldName, utf8Value); - } } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } @@ -306,9 +294,6 @@ public DocumentDimensions addLong(String fieldName, long value) { out.write((byte) 'l'); out.writeLong(value); add(fieldName, out.bytes()); - if (routingBuilder != null) { - routingBuilder.addMatching(fieldName, new BytesRef(Long.toString(value).getBytes(StandardCharsets.UTF_8))); - } } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } @@ -327,30 +312,12 @@ public DocumentDimensions addUnsignedLong(String fieldName, long value) { out.writeLong(value); } add(fieldName, out.bytes()); - if (routingBuilder != null) { - routingBuilder.addMatching(fieldName, new BytesRef(Long.toString(value).getBytes(StandardCharsets.UTF_8))); - } return this; } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } } - @Override - public DocumentDimensions addDouble(String fieldName, double value) { - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.write((byte) 'd'); - out.writeDouble(value); - add(fieldName, out.bytes()); - if (routingBuilder != null) { - routingBuilder.addMatching(fieldName, new BytesRef(Double.toString(value).getBytes(StandardCharsets.UTF_8))); - } - } catch (IOException e) { - throw new IllegalArgumentException("Dimension field cannot be serialized.", e); - } - return this; - } - @Override public DocumentDimensions validate(final IndexSettings settings) { if (settings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ID_HASHING) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 27e281ed7fb52..46b7181eace23 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -15,7 +15,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.hash.MurmurHash3.Hash128; import org.elasticsearch.common.util.ByteUtils; @@ -24,6 +23,7 @@ import org.elasticsearch.index.query.SearchExecutionContext; import java.util.Arrays; +import java.util.Base64; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -106,7 +106,7 @@ private TsidExtractingIdFieldMapper() { private static final long SEED = 0; - public static void createField(DocumentParserContext context, IndexRouting.ExtractFromSource.Builder routingBuilder, BytesRef tsid) { + public static void createField(DocumentParserContext context, BytesRef tsid) { List timestampFields = context.rootDoc().getFields(DataStreamTimestampFieldMapper.DEFAULT_PATH); if (timestampFields.isEmpty()) { throw new IllegalArgumentException( @@ -114,22 +114,7 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra ); } long timestamp = timestampFields.get(0).numericValue().longValue(); - byte[] suffix = new byte[16]; - String id = createId(context.hasDynamicMappers() == false, routingBuilder, tsid, timestamp, suffix); - /* - * Make sure that _id from extracting the tsid matches that _id - * from extracting the _source. This should be true for all valid - * documents with valid mappings. *But* some invalid mappings - * will not parse the field but be rejected later by the dynamic - * mappings machinery. So if there are any dynamic mappings - * at all we just skip the assertion because we can't be sure - * it always must pass. - */ - IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); - assert context.getDynamicMappers().isEmpty() == false - || context.getDynamicRuntimeFields().isEmpty() == false - || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); - + String id = createId(context.getShardId(), tsid, timestamp); if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { throw new IllegalArgumentException( String.format( @@ -147,30 +132,16 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); } - public static String createId( - boolean dynamicMappersExists, - IndexRouting.ExtractFromSource.Builder routingBuilder, - BytesRef tsid, - long timestamp, - byte[] suffix - ) { + public static String createId(int shardId, BytesRef tsid, long timestamp) { Hash128 hash = new Hash128(); MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); - ByteUtils.writeLongLE(hash.h1, suffix, 0); - ByteUtils.writeLongBE(timestamp, suffix, 8); // Big Ending shrinks the inverted index by ~37% - - String id = routingBuilder.createId(suffix, () -> { - if (dynamicMappersExists == false) { - throw new IllegalStateException( - "Didn't find any fields to include in the routing which would be fine if there are" - + " dynamic mapping waiting but we couldn't find any of those either!" - ); - } - return 0; - }); - assert Uid.isURLBase64WithoutPadding(id); // Make sure we get to use Uid's nice optimizations - return id; + byte[] bytes = new byte[20]; + ByteUtils.writeIntLE(shardId, bytes, 0); + ByteUtils.writeLongLE(hash.h1, bytes, 4); + ByteUtils.writeLongBE(timestamp, bytes, 12); // Big Ending shrinks the inverted index by ~37% + + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 86af6d21b7a09..d0dfcd5367bf7 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -359,7 +359,7 @@ public boolean hasNamedQueries() { * Parse a document with current mapping. */ public ParsedDocument parseDocument(SourceToParse source) throws DocumentParsingException { - return mapperService.documentParser().parseDocument(source, mappingLookup); + return mapperService.documentParser().parseDocument(source, mappingLookup, shardId); } public NestedLookup nestedLookup() { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index cf8a53f787c3b..f9cf4e3829a77 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -979,7 +979,8 @@ private Engine.IndexResult applyIndexOperation( isRetry, ifSeqNo, ifPrimaryTerm, - getRelativeTimeInNanos() + getRelativeTimeInNanos(), + shardId().id() ); Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { @@ -1016,7 +1017,8 @@ public static Engine.Index prepareIndex( boolean isRetry, long ifSeqNo, long ifPrimaryTerm, - long startTimeInNanos + long startTimeInNanos, + int shardId ) { assert source.dynamicTemplates().isEmpty() || origin == Engine.Operation.Origin.PRIMARY : "dynamic_templates parameter can only be associated with primary operations"; @@ -1026,7 +1028,7 @@ public static Engine.Index prepareIndex( documentMapper = DocumentMapper.createEmpty(mapperService); mapping = documentMapper.mapping(); } - ParsedDocument doc = documentMapper.parse(source); + ParsedDocument doc = documentMapper.parse(source, shardId); if (mapping != null) { // If we are indexing but there is no mapping we create one. This is to ensure that whenever at least a document is indexed // some mappings do exist. It covers for the case of indexing an empty doc (`{}`). diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index 74c9c93270471..dceaa20ba6d57 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -314,7 +314,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect ); DocumentParser documentParser = indexShard.mapperService().documentParser(); MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); - ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup); + ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup, indexShard.shardId().id()); // select the right fields and generate term vectors LuceneDocument doc = parsedDocument.rootDoc(); Set seenFields = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 7f9e808db9560..4d1f9b682e43e 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -19,7 +19,6 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchShardTask; import org.elasticsearch.action.search.SearchType; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -33,9 +32,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.mapper.IdLoader; -import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -76,8 +73,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.LongSupplier; @@ -897,23 +892,7 @@ public SourceLoader newSourceLoader() { @Override public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - var indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); - List routingPaths = indexService.getMetadata().getRoutingPaths(); - for (String routingField : routingPaths) { - if (routingField.contains("*")) { - // In case the routing fields include path matches, find any matches and add them as distinct fields - // to the routing path. - Set matchingRoutingPaths = new TreeSet<>(routingPaths); - for (Mapper mapper : indexService.mapperService().mappingLookup().fieldMappers()) { - if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.name())) { - matchingRoutingPaths.add(mapper.name()); - } - } - routingPaths = new ArrayList<>(matchingRoutingPaths); - break; - } - } - return IdLoader.createTsIdLoader(indexRouting, routingPaths); + return IdLoader.createTsIdLoader(request.shardId().id()); } else { return IdLoader.fromLeafStoredFieldLoader(); } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 51b2e62159a4d..d334b0f3ac333 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -727,7 +727,7 @@ private BytesRef parseBytesRefMap(Object value) { } Map m = (Map) value; - TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); + TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(); for (Map.Entry entry : m.entrySet()) { String f = entry.getKey().toString(); Object v = entry.getValue(); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java index d6c5fe812140f..6f57e4532a1fc 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java @@ -19,12 +19,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.DataStream; -import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; @@ -249,14 +245,6 @@ public void testTimeSeriesLoadDocIdAndVersion() throws Exception { } private static String createTSDBId(long timestamp) { - Settings.Builder b = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); - IndexMetadata indexMetadata = IndexMetadata.builder("idx").settings(b).numberOfShards(1).numberOfReplicas(0).build(); - IndexRouting.ExtractFromSource.Builder routingBuilder = ((IndexRouting.ExtractFromSource) IndexRouting.fromIndexMetadata( - indexMetadata - )).builder(); - routingBuilder.addMatching("field", new BytesRef("value")); - return createId(false, routingBuilder, new BytesRef("tsid"), timestamp, new byte[16]); + return createId(0, new BytesRef("tsid"), timestamp); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 72583b9aeb19e..3a714fca8fcce 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -70,7 +70,7 @@ private void withIndex(CheckedConsumer buildSource throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "text"))); withLuceneIndex(mapperService, writer -> { - ParsedDocument parsed = mapperService.documentParser().parseDocument(source(buildSource), mapperService.mappingLookup()); + ParsedDocument parsed = mapperService.documentParser().parseDocument(source(buildSource), mapperService.mappingLookup(), 0); writer.addDocuments(parsed.docs()); }, reader -> { assertThat(reader.leaves(), hasSize(1)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 5945e5c81856f..61ec5e61914e2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -47,9 +47,7 @@ public class IdLoaderTests extends ESTestCase { public void testSynthesizeIdSimple() throws Exception { - var routingPaths = List.of("dim1"); - var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + var idLoader = IdLoader.createTsIdLoader(7); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs = List.of( @@ -63,17 +61,17 @@ public void testSynthesizeIdSimple() throws Exception { assertThat(leafReader.numDocs(), equalTo(3)); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); // NOTE: time series data is ordered by (tsid, timestamp) - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs.get(2)))); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs.get(0)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs.get(1)))); + assertThat(leaf.getId(0), equalTo(expectedId(docs.get(2)))); + assertThat(leaf.getId(1), equalTo(expectedId(docs.get(0)))); + assertThat(leaf.getId(2), equalTo(expectedId(docs.get(1)))); }; - prepareIndexReader(indexAndForceMerge(routing, docs), verify, false); + prepareIndexReader(indexAndForceMerge(docs), verify, false); } public void testSynthesizeIdMultipleSegments() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + var idLoader = IdLoader.createTsIdLoader(7); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs1 = List.of( @@ -96,15 +94,15 @@ public void testSynthesizeIdMultipleSegments() throws Exception { ); CheckedConsumer buildIndex = writer -> { for (Doc doc : docs1) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc); } writer.flush(); for (Doc doc : docs2) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc); } writer.flush(); for (Doc doc : docs3) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc); } writer.flush(); }; @@ -115,22 +113,22 @@ public void testSynthesizeIdMultipleSegments() throws Exception { assertThat(leafReader.numDocs(), equalTo(docs1.size())); var leaf = idLoader.leaf(null, leafReader, IntStream.range(0, docs1.size()).toArray()); for (int i = 0; i < docs1.size(); i++) { - assertThat(leaf.getId(i), equalTo(expectedId(routing, docs1.get(i)))); + assertThat(leaf.getId(i), equalTo(expectedId(docs1.get(i)))); } } { LeafReader leafReader = indexReader.leaves().get(1).reader(); assertThat(leafReader.numDocs(), equalTo(docs2.size())); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 3 }); - assertThat(leaf.getId(0), equalTo(expectedId(routing, docs2.get(0)))); - assertThat(leaf.getId(3), equalTo(expectedId(routing, docs2.get(3)))); + assertThat(leaf.getId(0), equalTo(expectedId(docs2.get(0)))); + assertThat(leaf.getId(3), equalTo(expectedId(docs2.get(3)))); } { LeafReader leafReader = indexReader.leaves().get(2).reader(); assertThat(leafReader.numDocs(), equalTo(docs3.size())); var leaf = idLoader.leaf(null, leafReader, new int[] { 1, 2 }); - assertThat(leaf.getId(1), equalTo(expectedId(routing, docs3.get(1)))); - assertThat(leaf.getId(2), equalTo(expectedId(routing, docs3.get(2)))); + assertThat(leaf.getId(1), equalTo(expectedId(docs3.get(1)))); + assertThat(leaf.getId(2), equalTo(expectedId(docs3.get(2)))); } { LeafReader leafReader = indexReader.leaves().get(2).reader(); @@ -145,13 +143,14 @@ public void testSynthesizeIdMultipleSegments() throws Exception { public void testSynthesizeIdRandom() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(routing, routingPaths); + var idLoader = IdLoader.createTsIdLoader(7); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); Set expectedIDs = new HashSet<>(); List randomDocs = new ArrayList<>(); int numberOfTimeSeries = randomIntBetween(8, 64); for (int i = 0; i < numberOfTimeSeries; i++) { + long routingId = 0; int numberOfDimensions = randomIntBetween(1, 6); List dimensions = new ArrayList<>(numberOfDimensions); for (int j = 1; j <= numberOfDimensions; j++) { @@ -163,12 +162,13 @@ public void testSynthesizeIdRandom() throws Exception { value = randomAlphaOfLength(4); } dimensions.add(new Dimension(fieldName, value)); + routingId = value.hashCode(); } int numberOfSamples = randomIntBetween(1, 16); for (int j = 0; j < numberOfSamples; j++) { Doc doc = new Doc(startTime++, dimensions); randomDocs.add(doc); - expectedIDs.add(expectedId(routing, doc)); + expectedIDs.add(expectedId(doc)); } } CheckedConsumer verify = indexReader -> { @@ -181,14 +181,14 @@ public void testSynthesizeIdRandom() throws Exception { assertTrue("docId=" + i + " id=" + actualId, expectedIDs.remove(actualId)); } }; - prepareIndexReader(indexAndForceMerge(routing, randomDocs), verify, false); + prepareIndexReader(indexAndForceMerge(randomDocs), verify, false); assertThat(expectedIDs, empty()); } - private static CheckedConsumer indexAndForceMerge(IndexRouting.ExtractFromSource routing, List docs) { + private static CheckedConsumer indexAndForceMerge(List docs) { return writer -> { for (Doc doc : docs) { - indexDoc(routing, writer, doc); + indexDoc(writer, doc); } writer.forceMerge(1); }; @@ -220,8 +220,8 @@ private void prepareIndexReader( } } - private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter iw, Doc doc) throws IOException { - final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); + private static void indexDoc(IndexWriter iw, Doc doc) throws IOException { + final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); final List fields = new ArrayList<>(); fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); @@ -240,9 +240,8 @@ private static void indexDoc(IndexRouting.ExtractFromSource routing, IndexWriter iw.addDocument(fields); } - private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc) throws IOException { - var routingBuilder = routing.builder(); - var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routingBuilder); + private static String expectedId(Doc doc) throws IOException { + var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { timeSeriesIdBuilder.addLong(dimension.field, n.longValue()); @@ -250,13 +249,7 @@ private static String expectedId(IndexRouting.ExtractFromSource routing, Doc doc timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); } } - return TsidExtractingIdFieldMapper.createId( - false, - routingBuilder, - timeSeriesIdBuilder.buildTsidHash().toBytesRef(), - doc.timestamp, - new byte[16] - ); + return TsidExtractingIdFieldMapper.createId(7, timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp); } private static IndexRouting.ExtractFromSource createRouting(List routingPaths) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index c19c21d54a569..1a31655dc856a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -31,6 +31,9 @@ import static org.hamcrest.Matchers.equalTo; public class TsidExtractingIdFieldMapperTests extends MetadataMapperTestCase { + + private static final int SHARD_ID = 7; + private static class TestCase { private final String name; private final String expectedId; @@ -82,7 +85,7 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:00Z", - "XsFI2ajcFfi45iV3AAABfhMmioA", + "BwAAAKjcFfi45iV3AAABfhMmioA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -94,7 +97,7 @@ public static Iterable params() { items.add( new TestCase( "2022-01-01T01:00:01Z", - "XsFI2ajcFfi45iV3AAABfhMmjmg", + "BwAAAKjcFfi45iV3AAABfhMmjmg", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:01.000Z", b -> { @@ -106,7 +109,7 @@ public static Iterable params() { items.add( new TestCase( "1970-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3AAAAAAAAAAA", + "BwAAAKjcFfi45iV3AAAAAAAAAAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "1970-01-01T00:00:00.000Z", b -> { @@ -118,7 +121,7 @@ public static Iterable params() { items.add( new TestCase( "-9998-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3__6oggRgGAA", + "BwAAAKjcFfi45iV3__6oggRgGAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "-9998-01-01T00:00:00.000Z", b -> { @@ -130,7 +133,7 @@ public static Iterable params() { items.add( new TestCase( "9998-01-01T00:00:00Z", - "XsFI2ajcFfi45iV3AADmaSK9hAA", + "BwAAAKjcFfi45iV3AADmaSK9hAA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "9998-01-01T00:00:00.000Z", b -> { @@ -144,7 +147,7 @@ public static Iterable params() { items.add( new TestCase( "r1", - "XsFI2ajcFfi45iV3AAABfhMmioA", + "BwAAAKjcFfi45iV3AAABfhMmioA", "JJSLNivCxv3hDTQtWd6qGUwGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -180,7 +183,7 @@ public static Iterable params() { items.add( new TestCase( "r2", - "1y-UzR0iuE1-sOQpAAABfhMmioA", + "BwAAAB0iuE1-sOQpAAABfhMmioA", "JNY_frTR9GmCbhXgK4Y8W44GlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -192,7 +195,7 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcS1h1gf2J5a8AAABfhMmioA", + "BwAAAC1h1gf2J5a8AAABfhMmioA", "JEyfZsJIp3UNyfWG-4SjKFIGlT_5e6_NYGOZWULpmMG9IAlZlA", "2022-01-01T01:00:00.000Z", b -> { @@ -209,7 +212,7 @@ public static Iterable params() { items.add( new TestCase( "k1=dog", - "XsFI2SrEiVgZlSsYAAABfhMmioA", + "BwAAACrEiVgZlSsYAAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bipyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -222,7 +225,7 @@ public static Iterable params() { items.add( new TestCase( "k1=pumpkin", - "XsFI2W8GX8-0QcFxAAABfhMmioA", + "BwAAAG8GX8-0QcFxAAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bibzw1JBpU_-VsHjSz5HC1yy_swPEM1iGo", "2022-01-01T01:00:00.000Z", b -> { @@ -235,7 +238,7 @@ public static Iterable params() { items.add( new TestCase( "k1=empty string", - "XsFI2cna58i6D-Q6AAABfhMmioA", + "BwAAAMna58i6D-Q6AAABfhMmioA", "KJQKpjU9U63jhh-eNJ1f8bhaCD7uBpU_-SWGG0Uv9tZ1mLO2gi9rC1I", "2022-01-01T01:00:00.000Z", b -> { @@ -248,7 +251,7 @@ public static Iterable params() { items.add( new TestCase( "k2", - "XsFI2VqlzAuv-06kAAABfhMmioA", + "BwAAAFqlzAuv-06kAAABfhMmioA", "KB9H-tGrL_UzqMcqXcgBtzypyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -261,7 +264,7 @@ public static Iterable params() { items.add( new TestCase( "o.k3", - "XsFI2S_VhridAKDUAAABfhMmioA", + "BwAAAC_VhridAKDUAAABfhMmioA", "KGXATwN7ISd1_EycFRJ9h6qpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -274,7 +277,7 @@ public static Iterable params() { items.add( new TestCase( "o.r3", - "zh4dcUwfL7x__2oPAAABfhMmioA", + "BwAAAEwfL7x__2oPAAABfhMmioA", "KJaYZVZz8plfkEvvPBpi1EWpyU08BpU_-ZJxnTYtoe9Lsg-QvzL-qOY", "2022-01-01T01:00:00.000Z", b -> { @@ -305,7 +308,7 @@ public static Iterable params() { items.add( new TestCase( "L1=1", - "XsFI2fIe53BtV9PCAAABfhMmioA", + "BwAAAPIe53BtV9PCAAABfhMmioA", "KI4kVxcCLIMM2_VQGD575d-tm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -318,7 +321,7 @@ public static Iterable params() { items.add( new TestCase( "L1=min", - "XsFI2Qhu7hy1RoXRAAABfhMmioA", + "BwAAAAhu7hy1RoXRAAABfhMmioA", "KI4kVxcCLIMM2_VQGD575d8caJ3TBpU_-cLpg-VnCBnhYk33HZBle6E", "2022-01-01T01:00:00.000Z", b -> { @@ -331,7 +334,7 @@ public static Iterable params() { items.add( new TestCase( "L2=1234", - "XsFI2QTrNu7TTpc-AAABfhMmioA", + "BwAAAATrNu7TTpc-AAABfhMmioA", "KI_1WxF60L0IczG5ftUCWdndcGtgBpU_-QfM2BaR0DMagIfw3TDu_mA", "2022-01-01T01:00:00.000Z", b -> { @@ -344,7 +347,7 @@ public static Iterable params() { items.add( new TestCase( "o.L3=max", - "zh4dcWBQI6THHqxoAAABfhMmioA", + "BwAAAGBQI6THHqxoAAABfhMmioA", "KN4a6QzKhzc3nwzNLuZkV51xxTOVBpU_-erUU1qSW4eJ0kP0RmAB9TE", "2022-01-01T01:00:00.000Z", b -> { @@ -375,7 +378,7 @@ public static Iterable params() { items.add( new TestCase( "i1=1", - "XsFI2UMS_RWRoHYjAAABfhMmioA", + "BwAAAEMS_RWRoHYjAAABfhMmioA", "KLGFpvAV8QkWSmX54kXFMgitm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -388,7 +391,7 @@ public static Iterable params() { items.add( new TestCase( "i1=min", - "XsFI2adlQM5ILoA1AAABfhMmioA", + "BwAAAKdlQM5ILoA1AAABfhMmioA", "KLGFpvAV8QkWSmX54kXFMgjV8hFQBpU_-WG2MicRGWwJdBKWq2F4qy4", "2022-01-01T01:00:00.000Z", b -> { @@ -401,7 +404,7 @@ public static Iterable params() { items.add( new TestCase( "i2=1234", - "XsFI2bhxfB6J0kBFAAABfhMmioA", + "BwAAALhxfB6J0kBFAAABfhMmioA", "KJc4-5eN1uAlYuAknQQLUlxavn2sBpU_-UEXBjgaH1uYcbayrOhdgpc", "2022-01-01T01:00:00.000Z", b -> { @@ -414,7 +417,7 @@ public static Iterable params() { items.add( new TestCase( "o.i3=max", - "zh4dcelxKf19CbfdAAABfhMmioA", + "BwAAAOlxKf19CbfdAAABfhMmioA", "KKqnzPNBe8ObksSo8rNaIFPZPCcBBpU_-Rhd_U6Jn2pjQz2zpmBuJb4", "2022-01-01T01:00:00.000Z", b -> { @@ -445,7 +448,7 @@ public static Iterable params() { items.add( new TestCase( "s1=1", - "XsFI2Y_y-8kD_BFeAAABfhMmioA", + "BwAAAI_y-8kD_BFeAAABfhMmioA", "KFi_JDbvzWyAawmh8IEXedwGlT_5rZuNb-1ruHTTZhtsXRZpZRwWFoc", "2022-01-01T01:00:00.000Z", b -> { @@ -458,7 +461,7 @@ public static Iterable params() { items.add( new TestCase( "s1=min", - "XsFI2WV8VNVnmPVNAAABfhMmioA", + "BwAAAGV8VNVnmPVNAAABfhMmioA", "KFi_JDbvzWyAawmh8IEXedwGlT_5JgBZj9BSCms2_jgeFFhsmDlNFdM", "2022-01-01T01:00:00.000Z", b -> { @@ -471,7 +474,7 @@ public static Iterable params() { items.add( new TestCase( "s2=1234", - "XsFI2VO8mUr-J5CpAAABfhMmioA", + "BwAAAFO8mUr-J5CpAAABfhMmioA", "KKEQ2p3CkpMH61hNk_SuvI0GlT_53XBrYP5TPdmCR-vREPnt20e9f9w", "2022-01-01T01:00:00.000Z", b -> { @@ -484,7 +487,7 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dcQKh6K11zWeuAAABfhMmioA", + "BwAAAAKh6K11zWeuAAABfhMmioA", "KKVMoT_-GS95fvIBtR7XK9oGlT_5Dme9-H3sen0WZ7leJpCj7-vXau4", "2022-01-01T01:00:00.000Z", b -> { @@ -515,7 +518,7 @@ public static Iterable params() { items.add( new TestCase( "b1=1", - "XsFI2dKxqgT5JDQfAAABfhMmioA", + "BwAAANKxqgT5JDQfAAABfhMmioA", "KGPAUhTjWOsRfDmYp3SUELatm41vBpU_-TUExUU_bL3Puq_EBgIaLac", "2022-01-01T01:00:00.000Z", b -> { @@ -528,7 +531,7 @@ public static Iterable params() { items.add( new TestCase( "b1=min", - "XsFI2d_PD--DgUvoAAABfhMmioA", + "BwAAAN_PD--DgUvoAAABfhMmioA", "KGPAUhTjWOsRfDmYp3SUELYoK6qHBpU_-d8HkZFJ3aL2ZV1lgHAjT1g", "2022-01-01T01:00:00.000Z", b -> { @@ -541,7 +544,7 @@ public static Iterable params() { items.add( new TestCase( "b2=12", - "XsFI2aqX5QjiuhsEAAABfhMmioA", + "BwAAAKqX5QjiuhsEAAABfhMmioA", "KA58oUMzXeX1V5rh51Ste0K5K9vPBpU_-Wn8JQplO-x3CgoslYO5Vks", "2022-01-01T01:00:00.000Z", b -> { @@ -554,7 +557,7 @@ public static Iterable params() { items.add( new TestCase( "o.s3=max", - "zh4dccJ4YtN_21XHAAABfhMmioA", + "BwAAAMJ4YtN_21XHAAABfhMmioA", "KIwZH-StJBobjk9tCV-0OgjKmuwGBpU_-Sd-SdnoH3sbfKLgse-briE", "2022-01-01T01:00:00.000Z", b -> { @@ -585,7 +588,7 @@ public static Iterable params() { items.add( new TestCase( "ip1=192.168.0.1", - "XsFI2T5km9raIz_rAAABfhMmioA", + "BwAAAD5km9raIz_rAAABfhMmioA", "KNj6cLPRNEkqdjfOPIbg0wULrOlWBpU_-efWDsz6B6AnnwbZ7GeeocE", "2022-01-01T01:00:00.000Z", b -> { @@ -602,7 +605,7 @@ public static Iterable params() { items.add( new TestCase( "ip1=12.12.45.254", - "XsFI2QWfEH_e_6wIAAABfhMmioA", + "BwAAAAWfEH_e_6wIAAABfhMmioA", "KNj6cLPRNEkqdjfOPIbg0wVhJ08TBpU_-bANzLhvKPczlle7Pq0z8Qw", "2022-01-01T01:00:00.000Z", b -> { @@ -619,7 +622,7 @@ public static Iterable params() { items.add( new TestCase( "ip2=FE80:CD00:0000:0CDE:1257:0000:211E:729C", - "XsFI2WrrLHr1O4iQAAABfhMmioA", + "BwAAAGrrLHr1O4iQAAABfhMmioA", "KNDo3zGxO9HfN9XYJwKw2Z20h-WsBpU_-f4dSOLGSRlL1hoY2mgERuo", "2022-01-01T01:00:00.000Z", b -> { @@ -632,7 +635,7 @@ public static Iterable params() { items.add( new TestCase( "o.ip3=2001:db8:85a3:8d3:1319:8a2e:370:7348", - "zh4dca7d-9aKOS1MAAABfhMmioA", + "BwAAAK7d-9aKOS1MAAABfhMmioA", "KLXDcBBWJAjgJvjSdF_EJwraAQUzBpU_-ba6HZsIyKnGcbmc3KRLlmI", "2022-01-01T01:00:00.000Z", b -> { @@ -663,7 +666,7 @@ public static Iterable params() { items.add( new TestCase( "huge", - "WZKJR_dECvXBSl3xAAABfhMmioA", + "BwAAAPdECvXBSl3xAAABfhMmioA", "LIe18i0rRU_Bt9vB82F46LaS9mrUkvZq1K_2Gi7UEFMhFwNXrLA_H8TLpUr4", "2022-01-01T01:00:00.000Z", b -> { @@ -721,16 +724,15 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C source.accept(builder); builder.endObject(); SourceToParse sourceToParse = new SourceToParse(id, BytesReference.bytes(builder), builder.contentType()); - return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); + return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup(), 7); } } public void testRoutingPathCompliant() throws IOException { IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random()); IndexRouting indexRouting = createIndexSettings(version, indexSettings(version)).getIndexRouting(); - int indexShard = indexShard(indexRouting); - assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(indexShard)); - assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(indexShard)); + assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(SHARD_ID)); + assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(SHARD_ID)); } private int indexShard(IndexRouting indexRouting) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 82fb694db6c66..d4af486f34933 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -497,7 +497,8 @@ public Engine.Index createIndexOp(int docIdent) { false, UNASSIGNED_SEQ_NO, 0, - System.nanoTime() + System.nanoTime(), + 0 ); } } @@ -534,7 +535,8 @@ public Engine.Index createIndexOp(int docIdent) { false, UNASSIGNED_SEQ_NO, 0, - System.nanoTime() + System.nanoTime(), + 0 ); } } diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index 8c70b35d27c95..b6126fe324514 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -374,7 +374,7 @@ public void testParseZone() { } public void testParseTsid() throws IOException { - TimeSeriesIdBuilder timeSeriesIdBuilder = new TimeSeriesIdBuilder(null); + TimeSeriesIdBuilder timeSeriesIdBuilder = new TimeSeriesIdBuilder(); timeSeriesIdBuilder.addString("string", randomAlphaOfLength(10)); timeSeriesIdBuilder.addLong("long", randomLong()); timeSeriesIdBuilder.addUnsignedLong("ulong", randomLong()); diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index d4e9b1ad3b84c..ba573af8a60fc 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -107,7 +107,8 @@ public Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.O true, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, - System.nanoTime() + System.nanoTime(), + 0 ); return engineIndex; } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index d4c238322e28a..b5f59fc130977 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -67,7 +67,8 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou ), source, mappingLookup.getMapping().getRoot(), - ObjectMapper.Dynamic.getRootDynamic(mappingLookup) + ObjectMapper.Dynamic.getRootDynamic(mappingLookup), + 0 ); this.parser = parser; } diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index 885e02a8b5e6a..d2c408da55374 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -163,7 +163,7 @@ private List docs(long startTimestamp, String dim, long... values) thr } private static BytesReference tsid(String dim) throws IOException { - TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); idBuilder.addString("dim", dim); return idBuilder.buildTsidHash(); } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index 1ba9fa5d1d354..ddec29eaf95f0 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -352,7 +352,8 @@ private IndexReader initIndex(Directory directory, int size, int commitEvery) th ParsedDocument doc = mapperService.documentParser() .parseDocument( new SourceToParse("id" + d, BytesReference.bytes(source), XContentType.JSON), - mapperService.mappingLookup() + mapperService.mappingLookup(), + 0 ); writer.addDocuments(doc.docs()); diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index 0b76e786b26be..e3a611a50eace 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -800,7 +800,7 @@ private void assertGeoLine_TSDB( ArrayList points = testData.pointsForGroup(g); ArrayList timestamps = testData.timestampsForGroup(g); for (int i = 0; i < points.size(); i++) { - final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); + final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); builder.addString("group_id", testData.groups[g]); ArrayList fields = new ArrayList<>( Arrays.asList( From 846649b1d055fba15d2f40be979d19791986a89c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 13 Feb 2024 13:31:23 +0200 Subject: [PATCH 15/51] spotless fix --- .../elasticsearch/datastreams/TSDBPassthroughIndexingIT.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index dda114e5cd8f0..84f4e199030ec 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -187,10 +187,7 @@ public void testIndexingGettingAndSearching() throws Exception { ObjectPath.eval("properties.metricset", mapping), matchesMap().entry("type", "alias").entry("path", "attributes.metricset") ); - assertMap( - ObjectPath.eval("properties.number", mapping), - matchesMap().entry("type", "alias").entry("path", "attributes.number") - ); + assertMap(ObjectPath.eval("properties.number", mapping), matchesMap().entry("type", "alias").entry("path", "attributes.number")); assertMap( ObjectPath.eval("properties.pod.properties", mapping), matchesMap().extraOk().entry("name", matchesMap().entry("type", "alias").entry("path", "attributes.pod.name")) From f464b6e61f4c885e5c1e511c4725bb6fb9ea6a4e Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 13 Feb 2024 15:13:49 +0200 Subject: [PATCH 16/51] test fixes --- .../TSDBPassthroughIndexingIT.java | 18 ++++-- .../rest-api-spec/test/delete/70_tsdb.yml | 8 +-- .../test/tsdb/25_id_generation.yml | 56 +++++++++---------- .../TsidExtractingIdFieldMapperTests.java | 2 +- 4 files changed, 47 insertions(+), 37 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index 84f4e199030ec..d4de041d71260 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -91,7 +91,8 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase { "@timestamp": "$time", "attributes": { "metricset": "pod", - "number": $number, + "number.long": $number1, + "number.double": $number2, "pod": { "name": "$name", "uid": "$uid", @@ -146,7 +147,8 @@ public void testIndexingGettingAndSearching() throws Exception { DOC.replace("$time", formatInstant(time)) .replace("$uid", randomUUID()) .replace("$name", randomAlphaOfLength(4)) - .replace("$number", Long.toString(randomLong())) + .replace("$number1", Long.toString(randomLong())) + .replace("$number2", Double.toString(randomDouble())) .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))), XContentType.JSON ); @@ -178,7 +180,8 @@ public void testIndexingGettingAndSearching() throws Exception { ); @SuppressWarnings("unchecked") var attributes = (Map>) ObjectPath.eval("properties.attributes.properties", mapping); - assertMap(attributes.get("number"), matchesMap().entry("type", "long").entry("time_series_dimension", true)); + assertMap(attributes.get("number.long"), matchesMap().entry("type", "long").entry("time_series_dimension", true)); + assertMap(attributes.get("number.double"), matchesMap().entry("type", "float").entry("time_series_dimension", true)); assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "ip").entry("time_series_dimension", true)); assertMap(attributes.get("pod.uid"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true)); assertMap(attributes.get("pod.name"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true)); @@ -187,7 +190,14 @@ public void testIndexingGettingAndSearching() throws Exception { ObjectPath.eval("properties.metricset", mapping), matchesMap().entry("type", "alias").entry("path", "attributes.metricset") ); - assertMap(ObjectPath.eval("properties.number", mapping), matchesMap().entry("type", "alias").entry("path", "attributes.number")); + assertMap( + ObjectPath.eval("properties.number.properties.long", mapping), + matchesMap().entry("type", "alias").entry("path", "attributes.number.long") + ); + assertMap( + ObjectPath.eval("properties.number.properties.double", mapping), + matchesMap().entry("type", "alias").entry("path", "attributes.number.double") + ); assertMap( ObjectPath.eval("properties.pod.properties", mapping), matchesMap().extraOk().entry("name", matchesMap().entry("type", "alias").entry("path", "attributes.pod.name")) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index 83f8aab34e02a..cd8fc64c28165 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -43,15 +43,15 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + - match: { _id: AAAAAAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 1 } - do: delete: index: weather_sensors - id: crxuhAep5Npwt_etAAABiHD35_g - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + id: AAAAAAep5Npwt_etAAABiHD35_g + - match: { _id: AAAAAAep5Npwt_etAAABiHD35_g } - match: { result: deleted } - match: { _version: 2 } @@ -68,6 +68,6 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } + - match: { _id: AAAAAAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 3 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 6ef03ba8ebcc4..8e0269920f7cb 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + - match: {items.0.index._id: AQAAALB9sSWsyrL5AAABeRnS7fM} - do: bulk: @@ -85,7 +85,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} + - match: {items.0.index._id: AQAAALB9sSWsyrL5AAABeRnS7fM} - do: search: @@ -97,39 +97,39 @@ generates a consistent id: - match: {hits.total.value: 9} - - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY } + - match: { hits.hits.0._id: AQAAAMfoxSs_KdA5AAABeRnRFAY } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.1._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } + - match: { hits.hits.1._id: AQAAALB9sSWsyrL5AAABeRnRGTM } - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.2._id: cn4excfoxSs_KdA5AAABeRnRYiY } + - match: { hits.hits.2._id: AQAAAMfoxSs_KdA5AAABeRnRYiY } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } + - match: { hits.hits.3._id: AQAAALB9sSWsyrL5AAABeRnRZ1M } - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.4._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } + - match: { hits.hits.4._id: AQAAALB9sSWsyrL5AAABeRnRtXM } - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR11Y } + - match: { hits.hits.5._id: AQAAAMfoxSs_KdA5AAABeRnR11Y } - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.6._id: cn4excfoxSs_KdA5AAABeRnR_mY } + - match: { hits.hits.6._id: AQAAAMfoxSs_KdA5AAABeRnR_mY } - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } + - match: { hits.hits.7._id: AQAAALB9sSWsyrL5AAABeRnSA5M } - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.8._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } + - match: { hits.hits.8._id: AQAAALB9sSWsyrL5AAABeRnS7fM } - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } @@ -170,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: cn4excfoxSs_KdA5AAABeRnR_mY} + - match: {_id: AQAAAMfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -215,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: cn4excfoxSs_KdA5AAABeRnR_mY} + - match: {items.0.index._id: AQAAAMfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -239,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[cn4excfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[AQAAAMfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -268,7 +268,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[cn4excfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[AQAAAMfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -284,12 +284,12 @@ ids query: - field: k8s.pod.network.tx query: ids: - values: ["cn4excfoxSs_KdA5AAABeRnR11Y", "cn4excfoxSs_KdA5AAABeRnR_mY"] + values: ["AQAAAMfoxSs_KdA5AAABeRnR11Y", "AQAAAMfoxSs_KdA5AAABeRnR_mY"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "cn4excfoxSs_KdA5AAABeRnR11Y"} + - match: {hits.hits.0._id: "AQAAAMfoxSs_KdA5AAABeRnR11Y"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} - - match: {hits.hits.1._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: {hits.hits.1._id: "AQAAAMfoxSs_KdA5AAABeRnR_mY" } - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} --- @@ -301,9 +301,9 @@ get: - do: get: index: test - id: cZZNs7B9sSWsyrL5AAABeRnSA5M + id: AQAAALB9sSWsyrL5AAABeRnSA5M - match: {_index: test} - - match: {_id: cZZNs7B9sSWsyrL5AAABeRnSA5M} + - match: {_id: AQAAALB9sSWsyrL5AAABeRnSA5M} - match: _source: "@timestamp": "2021-04-28T18:51:04.467Z" @@ -351,7 +351,7 @@ delete: - do: delete: index: test - id: cn4excfoxSs_KdA5AAABeRnR_mY + id: AQAAAMfoxSs_KdA5AAABeRnR_mY - match: {result: deleted} --- @@ -391,20 +391,20 @@ delete over _bulk: mget: index: test body: - ids: [ cn4excfoxSs_KdA5AAABeRnR_mY, cn4excfoxSs_KdA5AAABeRnR11Y ] + ids: [ AQAAAMfoxSs_KdA5AAABeRnR_mY, AQAAAMfoxSs_KdA5AAABeRnR11Y ] - match: { docs.0._index: "test" } - - match: { docs.0._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } + - match: { docs.0._id: "AQAAAMfoxSs_KdA5AAABeRnR_mY" } - match: { docs.0.found: true } - match: { docs.1._index: "test" } - - match: { docs.1._id: "cn4excfoxSs_KdA5AAABeRnR11Y" } + - match: { docs.1._id: "AQAAAMfoxSs_KdA5AAABeRnR11Y" } - match: { docs.1.found: true } - do: bulk: index: test body: - - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR_mY"}}' - - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR11Y"}}' + - '{"delete": {"_id": "AQAAAMfoxSs_KdA5AAABeRnR_mY"}}' + - '{"delete": {"_id": "AQAAAMfoxSs_KdA5AAABeRnR11Y"}}' - '{"delete": {"_id": "not found ++ not found"}}' - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} @@ -454,7 +454,7 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: OcEOGchJrjH1fFX8AAABeRnRGTM} + - match: {items.0.index._id: AAAAAMhJrjH1fFX8AAABeRnRGTM} --- routing_path matches object: @@ -495,4 +495,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: 8bgiqW9JKwAyp1bZAAABeRnRGTM} + - match: {items.0.index._id: AAAAAG9JKwAyp1bZAAABeRnRGTM} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 1a31655dc856a..71c3930ae3e12 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -724,7 +724,7 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C source.accept(builder); builder.endObject(); SourceToParse sourceToParse = new SourceToParse(id, BytesReference.bytes(builder), builder.contentType()); - return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup(), 7); + return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup(), SHARD_ID); } } From 605cdf346fb9bb8baff04e470592843046e2b2e3 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 14 Feb 2024 10:37:18 +0200 Subject: [PATCH 17/51] index version guard --- .../timeseries/TimeSeriesAggregator.java | 2 +- .../timeseries/InternalTimeSeriesTests.java | 2 +- .../timeseries/TimeSeriesAggregatorTests.java | 2 +- .../org/elasticsearch/index/IndexMode.java | 4 +- .../elasticsearch/index/IndexVersions.java | 1 + .../elasticsearch/index/mapper/IdLoader.java | 45 +++++++++++++++-- .../index/mapper/TimeSeriesIdFieldMapper.java | 19 ++++++- .../mapper/TsidExtractingIdFieldMapper.java | 50 ++++++++++++++++++- .../search/DefaultSearchContext.java | 10 +++- .../elasticsearch/search/DocValueFormat.java | 2 +- .../index/mapper/IdLoaderTests.java | 10 ++-- .../search/DocValueFormatTests.java | 2 +- .../rate/TimeSeriesRateAggregatorTests.java | 2 +- .../aggregations/GeoLineAggregatorTests.java | 2 +- 14 files changed, 130 insertions(+), 23 deletions(-) diff --git a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java index d29a964b526f9..9cd7f7a86e532 100644 --- a/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java +++ b/modules/aggregations/src/main/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregator.java @@ -157,7 +157,7 @@ public void collect(int doc, long bucket) throws IOException { return; } - TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); + TimeSeriesIdFieldMapper.TimeSeriesIdBuilder tsidBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (TsidConsumer consumer : dimensionConsumers.values()) { consumer.accept(doc, tsidBuilder); } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java index c6cd161b879da..a77ea04a2d8a1 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/InternalTimeSeriesTests.java @@ -41,7 +41,7 @@ private List randomBuckets(boolean keyed, InternalAggregations a List> keys = randomKeys(bucketKeys(randomIntBetween(1, 4)), numberOfBuckets); for (int j = 0; j < numberOfBuckets; j++) { long docCount = randomLongBetween(0, Long.MAX_VALUE / (20L * numberOfBuckets)); - var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); + var builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (var entry : keys.get(j).entrySet()) { builder.addString(entry.getKey(), (String) entry.getValue()); } diff --git a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java index 8c10d55b9acd4..54d1c14931d92 100644 --- a/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java +++ b/modules/aggregations/src/test/java/org/elasticsearch/aggregations/bucket/timeseries/TimeSeriesAggregatorTests.java @@ -92,7 +92,7 @@ public static void writeTS(RandomIndexWriter iw, long timestamp, Object[] dimens final List fields = new ArrayList<>(); fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); fields.add(new LongPoint(DataStreamTimestampFieldMapper.DEFAULT_PATH, timestamp)); - final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(); + final TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); for (int i = 0; i < dimensions.length; i += 2) { if (dimensions[i + 1] instanceof Number n) { builder.addLong(dimensions[i].toString(), n.longValue()); diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 67bd322c87c53..05afc14e0f0cd 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -10,6 +10,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; @@ -196,7 +197,8 @@ public IdFieldMapper buildIdFieldMapper(BooleanSupplier fieldDataEnabled) { @Override public DocumentDimensions buildDocumentDimensions(IndexSettings settings) { - return new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); + IndexRouting.ExtractFromSource routing = (IndexRouting.ExtractFromSource) settings.getIndexRouting(); + return new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(routing.builder()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index ae6185cdcc6b6..4d89cdb66bf27 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -101,6 +101,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion NEW_INDEXVERSION_FORMAT = def(8_501_00_0, Version.LUCENE_9_9_1); public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); + public static final IndexVersion TIME_SERIES_SHARD_ID_IN_ID = def(8_503_00_0, Version.LUCENE_9_9_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index 9073fe7ccdafc..e268f00fc3a3a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -12,11 +12,14 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.index.fieldvisitor.LeafStoredFieldLoader; import java.io.IOException; +import java.util.List; /** * Responsible for loading the _id from stored fields or for TSDB synthesizing the _id from the routing, _tsid and @timestamp fields. @@ -33,8 +36,8 @@ static IdLoader fromLeafStoredFieldLoader() { /** * @return returns an {@link IdLoader} instance that syn synthesizes _id from routing, _tsid and @timestamp fields. */ - static IdLoader createTsIdLoader(int shardId) { - return new TsIdLoader(shardId); + static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths, int shardId) { + return new TsIdLoader(indexRouting, routingPaths, shardId); } Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException; @@ -54,13 +57,40 @@ sealed interface Leaf permits StoredLeaf, TsIdLeaf { final class TsIdLoader implements IdLoader { - final int shardId; + private final IndexRouting.ExtractFromSource indexRouting; + private final List routingPaths; + private final int shardId; - TsIdLoader(int shardId) { + TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths, int shardId) { + this.routingPaths = routingPaths; + this.indexRouting = indexRouting; this.shardId = shardId; } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { + IndexRouting.ExtractFromSource.Builder[] builders = null; + if (indexRouting != null) { + builders = new IndexRouting.ExtractFromSource.Builder[docIdsInLeaf.length]; + for (int i = 0; i < builders.length; i++) { + builders[i] = indexRouting.builder(); + } + + for (String routingField : routingPaths) { + // Routing field must always be keyword fields, so it is ok to use SortedSetDocValues directly here. + SortedSetDocValues dv = DocValues.getSortedSet(reader, routingField); + for (int i = 0; i < docIdsInLeaf.length; i++) { + int docId = docIdsInLeaf[i]; + var builder = builders[i]; + if (dv.advanceExact(docId)) { + for (int j = 0; j < dv.docValueCount(); j++) { + BytesRef routingValue = dv.lookupOrd(dv.nextOrd()); + builder.addMatching(routingField, routingValue); + } + } + } + } + } + String[] ids = new String[docIdsInLeaf.length]; // Each document always has exactly one tsid and one timestamp: SortedDocValues tsIdDocValues = DocValues.getSorted(reader, TimeSeriesIdFieldMapper.NAME); @@ -75,7 +105,12 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] assert found; assert timestampDocValues.docValueCount() == 1; long timestamp = timestampDocValues.nextValue(); - ids[i] = TsidExtractingIdFieldMapper.createId(shardId, tsid, timestamp); + if (builders != null) { + var routingBuilder = builders[i]; + ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); + } else { + ids[i] = TsidExtractingIdFieldMapper.createId(shardId, tsid, timestamp); + } } return new TsIdLeaf(docIdsInLeaf, ids); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 22e9b99d8877a..4ba46d308ebbd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -12,6 +12,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.hash.Murmur3Hasher; @@ -20,6 +21,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; @@ -137,7 +139,11 @@ public void postParse(DocumentParserContext context) throws IOException { ? timeSeriesIdBuilder.buildLegacyTsid().toBytesRef() : timeSeriesIdBuilder.buildTsidHash().toBytesRef(); context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); - TsidExtractingIdFieldMapper.createField(context, timeSeriesId); + TsidExtractingIdFieldMapper.createField( + context, + getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_SHARD_ID_IN_ID) ? timeSeriesIdBuilder.routingBuilder : null, + timeSeriesId + ); } private IndexVersion getIndexVersionCreated(final DocumentParserContext context) { @@ -181,10 +187,15 @@ private record Dimension(BytesRef name, BytesReference value) {} * to build the _tsid field for the document. */ private final SortedSet dimensions = new TreeSet<>(Comparator.comparing(o -> o.name)); - /** * Builds the routing. Used for building {@code _id}. If null then skipped. */ + @Nullable + private final IndexRouting.ExtractFromSource.Builder routingBuilder; + + public TimeSeriesIdBuilder(@Nullable IndexRouting.ExtractFromSource.Builder routingBuilder) { + this.routingBuilder = routingBuilder; + } public BytesReference buildLegacyTsid() throws IOException { if (dimensions.isEmpty()) { @@ -277,6 +288,10 @@ public DocumentDimensions addString(String fieldName, BytesRef utf8Value) { */ out.writeBytesRef(utf8Value); add(fieldName, out.bytes()); + + if (routingBuilder != null) { + routingBuilder.addMatching(fieldName, utf8Value); + } } catch (IOException e) { throw new IllegalArgumentException("Dimension field cannot be serialized.", e); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 46b7181eace23..28fc78dbb8014 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -15,6 +15,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.hash.MurmurHash3; import org.elasticsearch.common.hash.MurmurHash3.Hash128; import org.elasticsearch.common.util.ByteUtils; @@ -106,7 +107,7 @@ private TsidExtractingIdFieldMapper() { private static final long SEED = 0; - public static void createField(DocumentParserContext context, BytesRef tsid) { + public static void createField(DocumentParserContext context, IndexRouting.ExtractFromSource.Builder routingBuilder, BytesRef tsid) { List timestampFields = context.rootDoc().getFields(DataStreamTimestampFieldMapper.DEFAULT_PATH); if (timestampFields.isEmpty()) { throw new IllegalArgumentException( @@ -114,7 +115,26 @@ public static void createField(DocumentParserContext context, BytesRef tsid) { ); } long timestamp = timestampFields.get(0).numericValue().longValue(); - String id = createId(context.getShardId(), tsid, timestamp); + String id; + if (routingBuilder != null) { + byte[] suffix = new byte[16]; + id = createId(context.hasDynamicMappers() == false, routingBuilder, tsid, timestamp, suffix); + /* + * Make sure that _id from extracting the tsid matches that _id + * from extracting the _source. This should be true for all valid + * documents with valid mappings. *But* some invalid mappings + * will not parse the field but be rejected later by the dynamic + * mappings machinery. So if there are any dynamic mappings + * at all we just skip the assertion because we can't be sure + * it always must pass. + */ + IndexRouting.ExtractFromSource indexRouting = (IndexRouting.ExtractFromSource) context.indexSettings().getIndexRouting(); + assert context.getDynamicMappers().isEmpty() == false + || context.getDynamicRuntimeFields().isEmpty() == false + || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); + } else { + id = createId(context.getShardId(), tsid, timestamp); + } if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { throw new IllegalArgumentException( String.format( @@ -144,6 +164,32 @@ public static String createId(int shardId, BytesRef tsid, long timestamp) { return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); } + public static String createId( + boolean dynamicMappersExists, + IndexRouting.ExtractFromSource.Builder routingBuilder, + BytesRef tsid, + long timestamp, + byte[] suffix + ) { + Hash128 hash = new Hash128(); + MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); + + ByteUtils.writeLongLE(hash.h1, suffix, 0); + ByteUtils.writeLongBE(timestamp, suffix, 8); // Big Ending shrinks the inverted index by ~37% + + String id = routingBuilder.createId(suffix, () -> { + if (dynamicMappersExists == false) { + throw new IllegalStateException( + "Didn't find any fields to include in the routing which would be fine if there are" + + " dynamic mapping waiting but we couldn't find any of those either!" + ); + } + return 0; + }); + assert Uid.isURLBase64WithoutPadding(id); // Make sure we get to use Uid's nice optimizations + return id; + } + @Override public String documentDescription(DocumentParserContext context) { /* diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 4d1f9b682e43e..eac0b741dffbe 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -19,6 +19,7 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.action.search.SearchShardTask; import org.elasticsearch.action.search.SearchType; +import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Releasable; @@ -26,6 +27,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -892,7 +894,13 @@ public SourceLoader newSourceLoader() { @Override public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - return IdLoader.createTsIdLoader(request.shardId().id()); + IndexRouting.ExtractFromSource indexRouting = null; + List routingPaths = null; + if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_SHARD_ID_IN_ID)) { + indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); + routingPaths = indexService.getMetadata().getRoutingPaths(); + } + return IdLoader.createTsIdLoader(indexRouting, routingPaths, request.shardId().id()); } else { return IdLoader.fromLeafStoredFieldLoader(); } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index d334b0f3ac333..51b2e62159a4d 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -727,7 +727,7 @@ private BytesRef parseBytesRefMap(Object value) { } Map m = (Map) value; - TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(); + TimeSeriesIdBuilder builder = new TimeSeriesIdBuilder(null); for (Map.Entry entry : m.entrySet()) { String f = entry.getKey().toString(); Object v = entry.getValue(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 61ec5e61914e2..4deec725351c5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -47,7 +47,7 @@ public class IdLoaderTests extends ESTestCase { public void testSynthesizeIdSimple() throws Exception { - var idLoader = IdLoader.createTsIdLoader(7); + var idLoader = IdLoader.createTsIdLoader(null, null, 7); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs = List.of( @@ -71,7 +71,7 @@ public void testSynthesizeIdSimple() throws Exception { public void testSynthesizeIdMultipleSegments() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(7); + var idLoader = IdLoader.createTsIdLoader(null, null, 7); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs1 = List.of( @@ -143,7 +143,7 @@ public void testSynthesizeIdMultipleSegments() throws Exception { public void testSynthesizeIdRandom() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(7); + var idLoader = IdLoader.createTsIdLoader(null, null, 7); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); Set expectedIDs = new HashSet<>(); @@ -221,7 +221,7 @@ private void prepareIndexReader( } private static void indexDoc(IndexWriter iw, Doc doc) throws IOException { - final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); + final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); final List fields = new ArrayList<>(); fields.add(new SortedNumericDocValuesField(DataStreamTimestampFieldMapper.DEFAULT_PATH, doc.timestamp)); @@ -241,7 +241,7 @@ private static void indexDoc(IndexWriter iw, Doc doc) throws IOException { } private static String expectedId(Doc doc) throws IOException { - var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); + var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { timeSeriesIdBuilder.addLong(dimension.field, n.longValue()); diff --git a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java index b6126fe324514..8c70b35d27c95 100644 --- a/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java +++ b/server/src/test/java/org/elasticsearch/search/DocValueFormatTests.java @@ -374,7 +374,7 @@ public void testParseZone() { } public void testParseTsid() throws IOException { - TimeSeriesIdBuilder timeSeriesIdBuilder = new TimeSeriesIdBuilder(); + TimeSeriesIdBuilder timeSeriesIdBuilder = new TimeSeriesIdBuilder(null); timeSeriesIdBuilder.addString("string", randomAlphaOfLength(10)); timeSeriesIdBuilder.addLong("long", randomLong()); timeSeriesIdBuilder.addUnsignedLong("ulong", randomLong()); diff --git a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java index d2c408da55374..885e02a8b5e6a 100644 --- a/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java +++ b/x-pack/plugin/analytics/src/test/java/org/elasticsearch/xpack/analytics/rate/TimeSeriesRateAggregatorTests.java @@ -163,7 +163,7 @@ private List docs(long startTimestamp, String dim, long... values) thr } private static BytesReference tsid(String dim) throws IOException { - TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); + TimeSeriesIdFieldMapper.TimeSeriesIdBuilder idBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); idBuilder.addString("dim", dim); return idBuilder.buildTsidHash(); } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java index e3a611a50eace..0b76e786b26be 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/GeoLineAggregatorTests.java @@ -800,7 +800,7 @@ private void assertGeoLine_TSDB( ArrayList points = testData.pointsForGroup(g); ArrayList timestamps = testData.timestampsForGroup(g); for (int i = 0; i < points.size(); i++) { - final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(); + final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); builder.addString("group_id", testData.groups[g]); ArrayList fields = new ArrayList<>( Arrays.asList( From 8f56346c48e73d24287cf5ec5b70252bf7283b83 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 14 Feb 2024 11:08:19 +0200 Subject: [PATCH 18/51] test fix --- .../index/mapper/TsidExtractingIdFieldMapperTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 71c3930ae3e12..466271db0c9a4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -729,7 +730,7 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C } public void testRoutingPathCompliant() throws IOException { - IndexVersion version = IndexVersionUtils.randomCompatibleVersion(random()); + IndexVersion version = IndexVersions.TIME_SERIES_SHARD_ID_IN_ID; IndexRouting indexRouting = createIndexSettings(version, indexSettings(version)).getIndexRouting(); assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(SHARD_ID)); assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(SHARD_ID)); From 93787f9565f1bf61550500eda04bf717aac364f6 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 14 Feb 2024 11:41:57 +0200 Subject: [PATCH 19/51] revert shardId changes --- .../cluster/routing/IndexRouting.java | 3 +-- .../elasticsearch/index/IndexVersions.java | 2 +- .../index/engine/TranslogDirectoryReader.java | 3 +-- .../index/mapper/DocumentMapper.java | 6 +----- .../index/mapper/DocumentParser.java | 10 ++++------ .../index/mapper/DocumentParserContext.java | 19 ++++--------------- .../elasticsearch/index/mapper/IdLoader.java | 12 ++++++------ .../index/mapper/TimeSeriesIdFieldMapper.java | 2 +- .../mapper/TsidExtractingIdFieldMapper.java | 7 ++++--- .../index/query/SearchExecutionContext.java | 2 +- .../elasticsearch/index/shard/IndexShard.java | 8 +++----- .../index/termvectors/TermVectorsService.java | 2 +- .../search/DefaultSearchContext.java | 4 ++-- .../common/lucene/uid/VersionsTests.java | 2 +- .../index/mapper/BlockSourceReaderTests.java | 2 +- .../index/mapper/IdLoaderTests.java | 8 ++++---- .../TsidExtractingIdFieldMapperTests.java | 4 ++-- .../recovery/RecoverySourceHandlerTests.java | 6 ++---- .../index/engine/TranslogHandler.java | 3 +-- .../mapper/TestDocumentParserContext.java | 3 +-- .../ValuesSourceReaderOperatorTests.java | 3 +-- 21 files changed, 43 insertions(+), 68 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 7bf6149c6cc23..48851295175b2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -404,8 +404,7 @@ private int idToHash(String id) { if (idBytes.length < 4) { throw new ResourceNotFoundException("invalid id [{}] for index [{}] in time series mode", id, indexName); } - int shardId = ByteUtils.readIntLE(idBytes, 0); - return Math.floorMod(shardId, routingNumShards); + return hashToShardId(ByteUtils.readIntLE(idBytes, 0)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 4d89cdb66bf27..42bd055e3e257 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -101,7 +101,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion NEW_INDEXVERSION_FORMAT = def(8_501_00_0, Version.LUCENE_9_9_1); public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); - public static final IndexVersion TIME_SERIES_SHARD_ID_IN_ID = def(8_503_00_0, Version.LUCENE_9_9_2); + public static final IndexVersion TIME_SERIES_ROUTING_ID_IN_ID = def(8_503_00_0, Version.LUCENE_9_9_2); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index 956459119b712..708d042c91bf9 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -259,8 +259,7 @@ private LeafReader createInMemoryLeafReader() { Map.of(), DocumentSizeObserver.EMPTY_INSTANCE ), - mappingLookup, - shardId.id() + mappingLookup ); parsedDocs.updateSeqID(operation.seqNo(), operation.primaryTerm()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java index c5b26850b9753..9b3496acfd9f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java @@ -89,11 +89,7 @@ public MappingLookup mappers() { } public ParsedDocument parse(SourceToParse source) throws DocumentParsingException { - return parse(source, 0); - } - - public ParsedDocument parse(SourceToParse source, int shardId) throws DocumentParsingException { - return documentParser.parseDocument(source, mappingLookup, shardId); + return documentParser.parseDocument(source, mappingLookup); } public void validate(IndexSettings settings, boolean checkLimits) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 6959dfb882df0..58ccd6025013f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -67,7 +67,7 @@ public final class DocumentParser { * @return the parsed document * @throws DocumentParsingException whenever there's a problem parsing the document */ - public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingLookup, int shardId) throws DocumentParsingException { + public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingLookup) throws DocumentParsingException { if (source.source() != null && source.source().length() == 0) { throw new DocumentParsingException(new XContentLocation(0, 0), "failed to parse, document is empty"); } @@ -80,7 +80,7 @@ public ParsedDocument parseDocument(SourceToParse source, MappingLookup mappingL XContentHelper.createParser(parserConfiguration, source.source(), xContentType) ) ) { - context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser, shardId); + context = new RootDocumentParserContext(mappingLookup, mappingParserContext, source, parser); validateStart(context.parser()); MetadataFieldMapper[] metadataFieldsMappers = mappingLookup.getMapping().getSortedMetadataMappers(); internalParseDocument(metadataFieldsMappers, context); @@ -838,16 +838,14 @@ private static class RootDocumentParserContext extends DocumentParserContext { MappingLookup mappingLookup, MappingParserContext mappingParserContext, SourceToParse source, - XContentParser parser, - int shardId + XContentParser parser ) throws IOException { super( mappingLookup, mappingParserContext, source, mappingLookup.getMapping().getRoot(), - ObjectMapper.Dynamic.getRootDynamic(mappingLookup), - shardId + ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); if (mappingLookup.getMapping().getRoot().subobjects()) { this.parser = DotExpandingXContentParser.expandDots(parser, this.path); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 87eabe0f8fabf..66c5de61bcd92 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -115,8 +115,6 @@ public int get() { private final Set fieldsAppliedFromTemplates; private final Set copyToFields; - private final int shardId; - private DocumentParserContext( MappingLookup mappingLookup, MappingParserContext mappingParserContext, @@ -133,8 +131,7 @@ private DocumentParserContext( ObjectMapper.Dynamic dynamic, Set fieldsAppliedFromTemplates, Set copyToFields, - DynamicMapperSize dynamicMapperSize, - final int shardId + DynamicMapperSize dynamicMapperSize ) { this.mappingLookup = mappingLookup; this.mappingParserContext = mappingParserContext; @@ -152,7 +149,6 @@ private DocumentParserContext( this.fieldsAppliedFromTemplates = fieldsAppliedFromTemplates; this.copyToFields = copyToFields; this.dynamicMappersSize = dynamicMapperSize; - this.shardId = shardId; } private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, DocumentParserContext in) { @@ -172,8 +168,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, dynamic, in.fieldsAppliedFromTemplates, in.copyToFields, - in.dynamicMappersSize, - in.shardId + in.dynamicMappersSize ); } @@ -182,8 +177,7 @@ protected DocumentParserContext( MappingParserContext mappingParserContext, SourceToParse source, ObjectMapper parent, - ObjectMapper.Dynamic dynamic, - int shardId + ObjectMapper.Dynamic dynamic ) { this( mappingLookup, @@ -201,8 +195,7 @@ protected DocumentParserContext( dynamic, new HashSet<>(), new HashSet<>(), - new DynamicMapperSize(), - shardId + new DynamicMapperSize() ); } @@ -238,10 +231,6 @@ public final SourceToParse sourceToParse() { return this.sourceToParse; } - public int getShardId() { - return shardId; - } - /** * Add the given {@code field} to the set of ignored fields. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index e268f00fc3a3a..1ae9593e61eb1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -36,8 +36,8 @@ static IdLoader fromLeafStoredFieldLoader() { /** * @return returns an {@link IdLoader} instance that syn synthesizes _id from routing, _tsid and @timestamp fields. */ - static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths, int shardId) { - return new TsIdLoader(indexRouting, routingPaths, shardId); + static IdLoader createTsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { + return new TsIdLoader(indexRouting, routingPaths); } Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException; @@ -59,12 +59,10 @@ final class TsIdLoader implements IdLoader { private final IndexRouting.ExtractFromSource indexRouting; private final List routingPaths; - private final int shardId; - TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths, int shardId) { + TsIdLoader(IndexRouting.ExtractFromSource indexRouting, List routingPaths) { this.routingPaths = routingPaths; this.indexRouting = indexRouting; - this.shardId = shardId; } public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] docIdsInLeaf) throws IOException { @@ -109,7 +107,9 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] var routingBuilder = builders[i]; ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); } else { - ids[i] = TsidExtractingIdFieldMapper.createId(shardId, tsid, timestamp); + // A + int routingId = 0; // FIXME + ids[i] = TsidExtractingIdFieldMapper.createId(routingId, tsid, timestamp); } } return new TsIdLeaf(docIdsInLeaf, ids); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 4ba46d308ebbd..15f88f931ac07 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -141,7 +141,7 @@ public void postParse(DocumentParserContext context) throws IOException { context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); TsidExtractingIdFieldMapper.createField( context, - getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_SHARD_ID_IN_ID) ? timeSeriesIdBuilder.routingBuilder : null, + getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID) ? timeSeriesIdBuilder.routingBuilder : null, timeSeriesId ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 28fc78dbb8014..566fc31325f7b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -133,7 +133,8 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); } else { - id = createId(context.getShardId(), tsid, timestamp); + int routingId = 0; // FIXME + id = createId(routingId, tsid, timestamp); } if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { throw new IllegalArgumentException( @@ -152,12 +153,12 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); } - public static String createId(int shardId, BytesRef tsid, long timestamp) { + public static String createId(int routingId, BytesRef tsid, long timestamp) { Hash128 hash = new Hash128(); MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); byte[] bytes = new byte[20]; - ByteUtils.writeIntLE(shardId, bytes, 0); + ByteUtils.writeIntLE(routingId, bytes, 0); ByteUtils.writeLongLE(hash.h1, bytes, 4); ByteUtils.writeLongBE(timestamp, bytes, 12); // Big Ending shrinks the inverted index by ~37% diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index d0dfcd5367bf7..86af6d21b7a09 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -359,7 +359,7 @@ public boolean hasNamedQueries() { * Parse a document with current mapping. */ public ParsedDocument parseDocument(SourceToParse source) throws DocumentParsingException { - return mapperService.documentParser().parseDocument(source, mappingLookup, shardId); + return mapperService.documentParser().parseDocument(source, mappingLookup); } public NestedLookup nestedLookup() { diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 730c7be091166..3bafe139756fd 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -980,8 +980,7 @@ private Engine.IndexResult applyIndexOperation( isRetry, ifSeqNo, ifPrimaryTerm, - getRelativeTimeInNanos(), - shardId().id() + getRelativeTimeInNanos() ); Mapping update = operation.parsedDoc().dynamicMappingsUpdate(); if (update != null) { @@ -1018,8 +1017,7 @@ public static Engine.Index prepareIndex( boolean isRetry, long ifSeqNo, long ifPrimaryTerm, - long startTimeInNanos, - int shardId + long startTimeInNanos ) { assert source.dynamicTemplates().isEmpty() || origin == Engine.Operation.Origin.PRIMARY : "dynamic_templates parameter can only be associated with primary operations"; @@ -1029,7 +1027,7 @@ public static Engine.Index prepareIndex( documentMapper = DocumentMapper.createEmpty(mapperService); mapping = documentMapper.mapping(); } - ParsedDocument doc = documentMapper.parse(source, shardId); + ParsedDocument doc = documentMapper.parse(source); if (mapping != null) { // If we are indexing but there is no mapping we create one. This is to ensure that whenever at least a document is indexed // some mappings do exist. It covers for the case of indexing an empty doc (`{}`). diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index dcc4b796af9b1..b4c0b200eb143 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -315,7 +315,7 @@ private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVect ); DocumentParser documentParser = indexShard.mapperService().documentParser(); MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); - ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup, indexShard.shardId().id()); + ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup); // select the right fields and generate term vectors LuceneDocument doc = parsedDocument.rootDoc(); Set seenFields = new HashSet<>(); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index eac0b741dffbe..4fac628694fcd 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -896,11 +896,11 @@ public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { IndexRouting.ExtractFromSource indexRouting = null; List routingPaths = null; - if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_SHARD_ID_IN_ID)) { + if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); routingPaths = indexService.getMetadata().getRoutingPaths(); } - return IdLoader.createTsIdLoader(indexRouting, routingPaths, request.shardId().id()); + return IdLoader.createTsIdLoader(indexRouting, routingPaths); } else { return IdLoader.fromLeafStoredFieldLoader(); } diff --git a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java index 6f57e4532a1fc..011a23ddb0512 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/uid/VersionsTests.java @@ -245,6 +245,6 @@ public void testTimeSeriesLoadDocIdAndVersion() throws Exception { } private static String createTSDBId(long timestamp) { - return createId(0, new BytesRef("tsid"), timestamp); + return createId(randomInt(), new BytesRef("tsid"), timestamp); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 3a714fca8fcce..72583b9aeb19e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -70,7 +70,7 @@ private void withIndex(CheckedConsumer buildSource throws IOException { MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "text"))); withLuceneIndex(mapperService, writer -> { - ParsedDocument parsed = mapperService.documentParser().parseDocument(source(buildSource), mapperService.mappingLookup(), 0); + ParsedDocument parsed = mapperService.documentParser().parseDocument(source(buildSource), mapperService.mappingLookup()); writer.addDocuments(parsed.docs()); }, reader -> { assertThat(reader.leaves(), hasSize(1)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 4deec725351c5..9afde7c7e2b1f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -47,7 +47,7 @@ public class IdLoaderTests extends ESTestCase { public void testSynthesizeIdSimple() throws Exception { - var idLoader = IdLoader.createTsIdLoader(null, null, 7); + var idLoader = IdLoader.createTsIdLoader(null, null); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs = List.of( @@ -71,7 +71,7 @@ public void testSynthesizeIdSimple() throws Exception { public void testSynthesizeIdMultipleSegments() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(null, null, 7); + var idLoader = IdLoader.createTsIdLoader(null, null); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); List docs1 = List.of( @@ -143,7 +143,7 @@ public void testSynthesizeIdMultipleSegments() throws Exception { public void testSynthesizeIdRandom() throws Exception { var routingPaths = List.of("dim1"); var routing = createRouting(routingPaths); - var idLoader = IdLoader.createTsIdLoader(null, null, 7); + var idLoader = IdLoader.createTsIdLoader(null, null); long startTime = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parseMillis("2023-01-01T00:00:00Z"); Set expectedIDs = new HashSet<>(); @@ -249,7 +249,7 @@ private static String expectedId(Doc doc) throws IOException { timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); } } - return TsidExtractingIdFieldMapper.createId(7, timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp); + return TsidExtractingIdFieldMapper.createId(randomInt(), timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp); } private static IndexRouting.ExtractFromSource createRouting(List routingPaths) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 466271db0c9a4..5b49ed3446f65 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -725,12 +725,12 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C source.accept(builder); builder.endObject(); SourceToParse sourceToParse = new SourceToParse(id, BytesReference.bytes(builder), builder.contentType()); - return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup(), SHARD_ID); + return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); } } public void testRoutingPathCompliant() throws IOException { - IndexVersion version = IndexVersions.TIME_SERIES_SHARD_ID_IN_ID; + IndexVersion version = IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID; IndexRouting indexRouting = createIndexSettings(version, indexSettings(version)).getIndexRouting(); assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(SHARD_ID)); assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(SHARD_ID)); diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index d4af486f34933..82fb694db6c66 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -497,8 +497,7 @@ public Engine.Index createIndexOp(int docIdent) { false, UNASSIGNED_SEQ_NO, 0, - System.nanoTime(), - 0 + System.nanoTime() ); } } @@ -535,8 +534,7 @@ public Engine.Index createIndexOp(int docIdent) { false, UNASSIGNED_SEQ_NO, 0, - System.nanoTime(), - 0 + System.nanoTime() ); } } diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index d3e3331b3dc3d..423990999fabd 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -106,8 +106,7 @@ public Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.O true, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, - System.nanoTime(), - 0 + System.nanoTime() ); return engineIndex; } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index b5f59fc130977..d4c238322e28a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -67,8 +67,7 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou ), source, mappingLookup.getMapping().getRoot(), - ObjectMapper.Dynamic.getRootDynamic(mappingLookup), - 0 + ObjectMapper.Dynamic.getRootDynamic(mappingLookup) ); this.parser = parser; } diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index ddec29eaf95f0..1ba9fa5d1d354 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -352,8 +352,7 @@ private IndexReader initIndex(Directory directory, int size, int commitEvery) th ParsedDocument doc = mapperService.documentParser() .parseDocument( new SourceToParse("id" + d, BytesReference.bytes(source), XContentType.JSON), - mapperService.mappingLookup(), - 0 + mapperService.mappingLookup() ); writer.addDocuments(doc.docs()); From b09820205a319345c6054882f3e99808e9a112b3 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 14 Feb 2024 14:47:54 +0200 Subject: [PATCH 20/51] add routing id field mapper --- .../elasticsearch/action/DocWriteRequest.java | 7 +- .../action/bulk/BulkOperation.java | 3 +- .../action/delete/DeleteRequest.java | 7 +- .../action/index/IndexRequest.java | 9 +- .../action/update/UpdateRequest.java | 7 +- .../cluster/routing/IndexRouting.java | 17 +- .../elasticsearch/index/mapper/IdLoader.java | 7 +- .../TimeSeriesRoutingIdFieldMapper.java | 171 ++++++++++++++++++ .../mapper/TsidExtractingIdFieldMapper.java | 25 +-- .../elasticsearch/indices/IndicesModule.java | 2 + .../ESIndexLevelReplicationTestCase.java | 2 +- 11 files changed, 233 insertions(+), 24 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java index 7f3578ce9f16f..c4ac655caac6d 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java @@ -161,7 +161,12 @@ public interface DocWriteRequest extends IndicesRequest, Accountable { /** * Finalize the request before executing or routing it. */ - void process(IndexRouting indexRouting); + void preProcess(IndexRouting indexRouting); + + /** + * Finalize the request after executing or routing it. + */ + void postProcess(IndexRouting indexRouting); /** * Pick the appropriate shard id to receive this request. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index 1d95f430d5c7e..70e7e642cf889 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -162,8 +162,9 @@ private Map> groupRequestsByShards(ClusterState c continue; } IndexRouting indexRouting = concreteIndices.routing(concreteIndex); - docWriteRequest.process(indexRouting); + docWriteRequest.preProcess(indexRouting); int shardId = docWriteRequest.route(indexRouting); + docWriteRequest.postProcess(indexRouting); List shardRequests = requestsByShard.computeIfAbsent( new ShardId(concreteIndex, shardId), shard -> new ArrayList<>() diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index e4b8edea58114..6eb7bcac64861 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -237,7 +237,12 @@ public boolean isRequireDataStream() { } @Override - public void process(IndexRouting indexRouting) { + public void preProcess(IndexRouting indexRouting) { + // Nothing to do + } + + @Override + public void postProcess(IndexRouting indexRouting) { // Nothing to do } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 03a99969a7deb..e1823a5a0b0b1 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -684,8 +684,13 @@ public VersionType versionType() { } @Override - public void process(IndexRouting indexRouting) { - indexRouting.process(this); + public void preProcess(IndexRouting indexRouting) { + indexRouting.preProcess(this); + } + + @Override + public void postProcess(IndexRouting indexRouting) { + indexRouting.postProcess(this); } /** diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 36b6cc6aa9964..9f7b7bf0720b5 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -848,7 +848,12 @@ public boolean isRequireDataStream() { } @Override - public void process(IndexRouting indexRouting) { + public void preProcess(IndexRouting indexRouting) { + // Nothing to do + } + + @Override + public void postProcess(IndexRouting indexRouting) { // Nothing to do } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 48851295175b2..9ec303762f708 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -68,7 +68,9 @@ private IndexRouting(IndexMetadata metadata) { this.routingFactor = metadata.getRoutingFactor(); } - public abstract void process(IndexRequest indexRequest); + public void preProcess(IndexRequest indexRequest) {} + + public void postProcess(IndexRequest indexRequest) {} /** * Called when indexing a document to generate the shard id that should contain @@ -141,7 +143,7 @@ private abstract static class IdAndRoutingOnly extends IndexRouting { protected abstract int shardId(String id, @Nullable String routing); @Override - public void process(IndexRequest indexRequest) { + public void preProcess(IndexRequest indexRequest) { if ("".equals(indexRequest.id())) { throw new IllegalArgumentException("if _id is specified it must not be empty"); } @@ -238,6 +240,8 @@ public static class ExtractFromSource extends IndexRouting { private final Predicate isRoutingPath; private final XContentParserConfiguration parserConfig; + private int routingId = 0; + ExtractFromSource(IndexMetadata metadata) { super(metadata); if (metadata.isRoutingPartitionedIndex()) { @@ -249,7 +253,13 @@ public static class ExtractFromSource extends IndexRouting { } @Override - public void process(IndexRequest indexRequest) {} + public void postProcess(IndexRequest indexRequest) { + if (indexRequest.id() == null) { + byte[] bytes = new byte[4]; + ByteUtils.writeIntLE(routingId, bytes, 0); + indexRequest.id(Base64.getUrlEncoder().encodeToString(bytes)); + } + } @Override public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { @@ -363,6 +373,7 @@ private int buildHash(IntSupplier onEmpty) { hash = 31 * hash + thisHash; prev = next; } + routingId = hash; return hash; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index 1ae9593e61eb1..4c4d43fc48575 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -107,8 +107,11 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] var routingBuilder = builders[i]; ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); } else { - // A - int routingId = 0; // FIXME + SortedNumericDocValues routingIdDocValues = DocValues.getSortedNumeric(reader, TimeSeriesRoutingIdFieldMapper.NAME); + found = routingIdDocValues.advanceExact(docId); + assert found; + assert routingIdDocValues.docValueCount() == 1 : routingIdDocValues.docValueCount(); + int routingId = Math.toIntExact(routingIdDocValues.nextValue()); ids[i] = TsidExtractingIdFieldMapper.createId(routingId, tsid, timestamp); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java new file mode 100644 index 0000000000000..6aa8700a12125 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java @@ -0,0 +1,171 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.util.ByteUtils; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.fielddata.FieldDataContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; +import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.field.IntegerDocValuesField; + +import java.util.Base64; +import java.util.Collection; +import java.util.Collections; + +/** + * Mapper for the {@code _seq_no} field. + * + * We expect to use the seq# for sorting, during collision checking and for + * doing range searches. Therefore the {@code _seq_no} field is stored both + * as a numeric doc value and as numeric indexed field. + * + * This mapper also manages the primary term field, which has no ES named + * equivalent. The primary term is only used during collision after receiving + * identical seq# values for two document copies. The primary term is stored as + * a doc value field without being indexed, since it is only intended for use + * as a key-value lookup. + + */ +public class TimeSeriesRoutingIdFieldMapper extends MetadataFieldMapper { + + public static final String NAME = "_routing_id"; + + public static final TimeSeriesRoutingIdFieldMapper INSTANCE = new TimeSeriesRoutingIdFieldMapper(); + + public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); + + static final class TimeSeriesRoutingIdFieldType extends SimpleMappedFieldType { + + private static final TimeSeriesRoutingIdFieldType INSTANCE = new TimeSeriesRoutingIdFieldType(); + + private TimeSeriesRoutingIdFieldType() { + super(NAME, true, false, true, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, Collections.emptyMap()); + } + + @Override + public String typeName() { + return NAME; + } + + private static int parse(Object value) { + if (value instanceof Number) { + return ((Number) value).intValue(); + } + if (value instanceof BytesRef) { + value = ((BytesRef) value).utf8ToString(); + } + return Integer.parseInt(value.toString()); + } + + @Override + public boolean mayExistInIndex(SearchExecutionContext context) { + return false; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); + } + + @Override + public Query termQuery(Object value, @Nullable SearchExecutionContext context) { + int v = parse(value); + return IntPoint.newExactQuery(name(), v); + } + + @Override + public Query termsQuery(Collection values, @Nullable SearchExecutionContext context) { + int[] v = values.stream().mapToInt(TimeSeriesRoutingIdFieldType::parse).toArray(); + return IntPoint.newSetQuery(name(), v); + } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + int l = Integer.MIN_VALUE; + int u = Integer.MAX_VALUE; + if (lowerTerm != null) { + l = parse(lowerTerm); + if (includeLower == false) { + if (l == Integer.MAX_VALUE) { + return new MatchNoDocsQuery(); + } + ++l; + } + } + if (upperTerm != null) { + u = parse(upperTerm); + if (includeUpper == false) { + if (u == Integer.MIN_VALUE) { + return new MatchNoDocsQuery(); + } + --u; + } + } + return IntPoint.newRangeQuery(name(), l, u); + } + + @Override + public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { + failIfNoDocValues(); + return new SortedNumericIndexFieldData.Builder(name(), NumericType.INT, IntegerDocValuesField::new); + } + } + + private TimeSeriesRoutingIdFieldMapper() { + super(TimeSeriesRoutingIdFieldType.INSTANCE); + } + + @Override + public void postParse(DocumentParserContext context) { + if (context.indexSettings().getMode() == IndexMode.TIME_SERIES + && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { + int routingId = decode(context.sourceToParse().id()); + var field = new IntField(NAME, routingId, Field.Store.YES); + context.doc().add(field); + } + } + + @Override + protected String contentType() { + return NAME; + } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } + + public static String encode(int routingId) { + byte[] bytes = new byte[4]; + ByteUtils.writeIntLE(routingId, bytes, 0); + return Base64.getUrlEncoder().encodeToString(bytes); + } + + public static int decode(String routingId) { + byte[] bytes = Base64.getUrlDecoder().decode(routingId); + return ByteUtils.readIntLE(bytes, 0); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 566fc31325f7b..ca8d7d8a23afb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -132,21 +132,22 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra assert context.getDynamicMappers().isEmpty() == false || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); + + if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "_id must be unset or set to [%s] but was [%s] because [%s] is in time_series mode", + id, + context.sourceToParse().id(), + context.indexSettings().getIndexMetadata().getIndex().getName() + ) + ); + } } else { - int routingId = 0; // FIXME + int routingId = TimeSeriesRoutingIdFieldMapper.decode(context.sourceToParse().id()); id = createId(routingId, tsid, timestamp); } - if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "_id must be unset or set to [%s] but was [%s] because [%s] is in time_series mode", - id, - context.sourceToParse().id(), - context.indexSettings().getIndexMetadata().getIndex().getName() - ) - ); - } context.id(id); BytesRef uidEncoded = Uid.encodeId(context.id()); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 795ed2120b098..041d68261d9cc 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -60,6 +60,7 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -247,6 +248,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(IdFieldMapper.NAME, IdFieldMapper.PARSER); builtInMetadataMappers.put(RoutingFieldMapper.NAME, RoutingFieldMapper.PARSER); builtInMetadataMappers.put(TimeSeriesIdFieldMapper.NAME, TimeSeriesIdFieldMapper.PARSER); + builtInMetadataMappers.put(TimeSeriesRoutingIdFieldMapper.NAME, TimeSeriesRoutingIdFieldMapper.PARSER); builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER); builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER); builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER); diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index ba6d7e441ef4a..287e3b4350d7c 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -872,7 +872,7 @@ private void executeShardBulkOnPrimary( ) { for (BulkItemRequest itemRequest : request.items()) { if (itemRequest.request() instanceof IndexRequest) { - ((IndexRequest) itemRequest.request()).process(primary.indexSettings().getIndexRouting()); + itemRequest.request().preProcess(primary.indexSettings().getIndexRouting()); } } final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); From 916e7d5d51aa12c9469ff35289ec281c7865b353 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Wed, 14 Feb 2024 14:51:15 +0200 Subject: [PATCH 21/51] Update docs/changelog/105501.yaml --- docs/changelog/105501.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/105501.yaml diff --git a/docs/changelog/105501.yaml b/docs/changelog/105501.yaml new file mode 100644 index 0000000000000..49691bb5d82f1 --- /dev/null +++ b/docs/changelog/105501.yaml @@ -0,0 +1,6 @@ +pr: 105501 +summary: Support non-keyword fields as dimension subfields of pass-through object + - with `TimeSeriesRoutingIdFieldMapper` +area: TSDB +type: enhancement +issues: [] From 7ea7febdc8beac8986372c32bd1d3b7d108b0e5a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 14 Feb 2024 15:00:45 +0200 Subject: [PATCH 22/51] fix shrunk test --- .../elasticsearch/datastreams/TSDBPassthroughIndexingIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index 4bac61aa436e6..c7b01abd02b11 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -249,6 +249,8 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { DOC.replace("$time", formatInstant(time)) .replace("$uid", randomUUID()) .replace("$name", randomAlphaOfLength(4)) + .replace("$number1", Long.toString(randomLong())) + .replace("$number2", Double.toString(randomDouble())) .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))), XContentType.JSON ); From 6ab5a3fc5983abea87c962c8334ef07b36982423 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 14 Feb 2024 21:44:18 +0200 Subject: [PATCH 23/51] use SortedDocValuesField --- .../cluster/routing/IndexRouting.java | 5 +- .../elasticsearch/index/mapper/IdLoader.java | 6 +- .../TimeSeriesRoutingIdFieldMapper.java | 97 +++++-------------- 3 files changed, 28 insertions(+), 80 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 9ec303762f708..5259dc9ebb209 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParser.Token; @@ -255,9 +256,7 @@ public static class ExtractFromSource extends IndexRouting { @Override public void postProcess(IndexRequest indexRequest) { if (indexRequest.id() == null) { - byte[] bytes = new byte[4]; - ByteUtils.writeIntLE(routingId, bytes, 0); - indexRequest.id(Base64.getUrlEncoder().encodeToString(bytes)); + indexRequest.id(TimeSeriesRoutingIdFieldMapper.encode(routingId)); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index 4c4d43fc48575..5d76fc86733f8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -107,11 +107,11 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] var routingBuilder = builders[i]; ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); } else { - SortedNumericDocValues routingIdDocValues = DocValues.getSortedNumeric(reader, TimeSeriesRoutingIdFieldMapper.NAME); + SortedDocValues routingIdDocValues = DocValues.getSorted(reader, TimeSeriesRoutingIdFieldMapper.NAME); found = routingIdDocValues.advanceExact(docId); assert found; - assert routingIdDocValues.docValueCount() == 1 : routingIdDocValues.docValueCount(); - int routingId = Math.toIntExact(routingIdDocValues.nextValue()); + BytesRef routingIdBytes = routingIdDocValues.lookupOrd(routingIdDocValues.ordValue()); + int routingId = TimeSeriesRoutingIdFieldMapper.decode(Uid.decodeId(routingIdBytes.bytes)); ids[i] = TsidExtractingIdFieldMapper.createId(routingId, tsid, timestamp); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java index 6aa8700a12125..833283408ee6f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java @@ -8,25 +8,21 @@ package org.elasticsearch.index.mapper; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.IntField; -import org.apache.lucene.document.IntPoint; -import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.search.Query; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.ByteUtils; -import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; -import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.plain.SortedOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.script.field.IntegerDocValuesField; +import org.elasticsearch.script.field.DelegateDocValuesField; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import java.util.Base64; -import java.util.Collection; import java.util.Collections; /** @@ -51,12 +47,12 @@ public class TimeSeriesRoutingIdFieldMapper extends MetadataFieldMapper { public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); - static final class TimeSeriesRoutingIdFieldType extends SimpleMappedFieldType { + static final class TimeSeriesRoutingIdFieldType extends MappedFieldType { private static final TimeSeriesRoutingIdFieldType INSTANCE = new TimeSeriesRoutingIdFieldType(); private TimeSeriesRoutingIdFieldType() { - super(NAME, true, false, true, TextSearchInfo.SIMPLE_MATCH_WITHOUT_TERMS, Collections.emptyMap()); + super(NAME, false, false, true, TextSearchInfo.NONE, Collections.emptyMap()); } @Override @@ -64,73 +60,27 @@ public String typeName() { return NAME; } - private static int parse(Object value) { - if (value instanceof Number) { - return ((Number) value).intValue(); - } - if (value instanceof BytesRef) { - value = ((BytesRef) value).utf8ToString(); - } - return Integer.parseInt(value.toString()); - } - - @Override - public boolean mayExistInIndex(SearchExecutionContext context) { - return false; - } - @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - throw new UnsupportedOperationException("Cannot fetch values for internal field [" + name() + "]."); - } - - @Override - public Query termQuery(Object value, @Nullable SearchExecutionContext context) { - int v = parse(value); - return IntPoint.newExactQuery(name(), v); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, MappedFieldType.FielddataOperation.SEARCH)); } @Override - public Query termsQuery(Collection values, @Nullable SearchExecutionContext context) { - int[] v = values.stream().mapToInt(TimeSeriesRoutingIdFieldType::parse).toArray(); - return IntPoint.newSetQuery(name(), v); - } - - @Override - public Query rangeQuery( - Object lowerTerm, - Object upperTerm, - boolean includeLower, - boolean includeUpper, - SearchExecutionContext context - ) { - int l = Integer.MIN_VALUE; - int u = Integer.MAX_VALUE; - if (lowerTerm != null) { - l = parse(lowerTerm); - if (includeLower == false) { - if (l == Integer.MAX_VALUE) { - return new MatchNoDocsQuery(); - } - ++l; - } - } - if (upperTerm != null) { - u = parse(upperTerm); - if (includeUpper == false) { - if (u == Integer.MIN_VALUE) { - return new MatchNoDocsQuery(); - } - --u; - } - } - return IntPoint.newRangeQuery(name(), l, u); + public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { + failIfNoDocValues(); + return new SortedOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new DelegateDocValuesField( + new ScriptDocValues.Strings(new ScriptDocValues.StringsSupplier(FieldData.toString(dv))), + n + ) + ); } @Override - public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - failIfNoDocValues(); - return new SortedNumericIndexFieldData.Builder(name(), NumericType.INT, IntegerDocValuesField::new); + public Query termQuery(Object value, SearchExecutionContext context) { + throw new IllegalArgumentException("[" + NAME + "] is not searchable"); } } @@ -142,8 +92,7 @@ private TimeSeriesRoutingIdFieldMapper() { public void postParse(DocumentParserContext context) { if (context.indexSettings().getMode() == IndexMode.TIME_SERIES && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { - int routingId = decode(context.sourceToParse().id()); - var field = new IntField(NAME, routingId, Field.Store.YES); + var field = new SortedDocValuesField(NAME, Uid.encodeId(context.sourceToParse().id())); context.doc().add(field); } } @@ -161,7 +110,7 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { public static String encode(int routingId) { byte[] bytes = new byte[4]; ByteUtils.writeIntLE(routingId, bytes, 0); - return Base64.getUrlEncoder().encodeToString(bytes); + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); } public static int decode(String routingId) { From 2f27478e7e20c6625c215048de1598ec6a29faea Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 15 Feb 2024 17:00:59 +0200 Subject: [PATCH 24/51] fix tests --- .../action/PainlessExecuteAction.java | 3 +- .../rest-api-spec/test/delete/70_tsdb.yml | 8 +-- .../test/nodes.stats/11_indices_metrics.yml | 33 +++++------ .../test/tsdb/25_id_generation.yml | 56 +++++++++---------- .../TimeSeriesRoutingIdFieldMapper.java | 2 +- .../cluster/routing/IndexRoutingTests.java | 10 ++-- .../TsidExtractingIdFieldMapperTests.java | 42 +++++--------- 7 files changed, 71 insertions(+), 83 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 3b67b76f59a31..f0f6d42af4095 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -60,6 +60,7 @@ import org.elasticsearch.index.mapper.OnScriptError; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; @@ -780,7 +781,7 @@ private static Response prepareRamIndex( XContentType xContentType = request.contextSetup.xContentType; String id; if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - id = null; // The id gets auto generated for time series indices. + id = TimeSeriesRoutingIdFieldMapper.encode(0); } else { id = "_id"; } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml index cd8fc64c28165..83f8aab34e02a 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/delete/70_tsdb.yml @@ -43,15 +43,15 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: AAAAAAep5Npwt_etAAABiHD35_g } + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 1 } - do: delete: index: weather_sensors - id: AAAAAAep5Npwt_etAAABiHD35_g - - match: { _id: AAAAAAep5Npwt_etAAABiHD35_g } + id: crxuhAep5Npwt_etAAABiHD35_g + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: deleted } - match: { _version: 2 } @@ -68,6 +68,6 @@ location: swamp temperature: 32.4 humidity: 88.9 - - match: { _id: AAAAAAep5Npwt_etAAABiHD35_g } + - match: { _id: crxuhAep5Npwt_etAAABiHD35_g } - match: { result: created } - match: { _version: 3 } diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index b119a1a1d94f3..3536c20071ab7 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -483,28 +483,29 @@ # 11. _source # 12. _tier # 13. _version - # 14. @timestamp - # 15. authors.age - # 16. authors.company - # 17. authors.company.keyword - # 18. authors.name.last_name - # 19. authors.name.first_name - # 20. authors.name.full_name - # 21. link - # 22. title - # 23. url + # 14. _routing_id + # 15. @timestamp + # 16. authors.age + # 17. authors.company + # 18. authors.company.keyword + # 19. authors.name.last_name + # 20. authors.name.first_name + # 21. authors.name.full_name + # 22. link + # 23. title + # 24. url # Object mappers: - # 24. authors - # 25. authors.name + # 25. authors + # 26. authors.name # Runtime field mappers: - # 26. a_source_field + # 27. a_source_field - gte: { nodes.$node_id.indices.mappings.total_count: 26 } - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead - - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 } - - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 26 } + - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 27648 } + - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 27 } - is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead - - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 26624 } + - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 27648 } --- "indices mappings does not exist in shards level": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml index 8e0269920f7cb..6ef03ba8ebcc4 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/25_id_generation.yml @@ -76,7 +76,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: AQAAALB9sSWsyrL5AAABeRnS7fM} + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} - do: bulk: @@ -85,7 +85,7 @@ generates a consistent id: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:52:04.467Z", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}' - - match: {items.0.index._id: AQAAALB9sSWsyrL5AAABeRnS7fM} + - match: {items.0.index._id: cZZNs7B9sSWsyrL5AAABeRnS7fM} - do: search: @@ -97,39 +97,39 @@ generates a consistent id: - match: {hits.total.value: 9} - - match: { hits.hits.0._id: AQAAAMfoxSs_KdA5AAABeRnRFAY } + - match: { hits.hits.0._id: cn4excfoxSs_KdA5AAABeRnRFAY } - match: { hits.hits.0._source.@timestamp: 2021-04-28T18:50:03.142Z } - match: { hits.hits.0._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.1._id: AQAAALB9sSWsyrL5AAABeRnRGTM } + - match: { hits.hits.1._id: cZZNs7B9sSWsyrL5AAABeRnRGTM } - match: { hits.hits.1._source.@timestamp: 2021-04-28T18:50:04.467Z } - match: { hits.hits.1._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.2._id: AQAAAMfoxSs_KdA5AAABeRnRYiY } + - match: { hits.hits.2._id: cn4excfoxSs_KdA5AAABeRnRYiY } - match: { hits.hits.2._source.@timestamp: 2021-04-28T18:50:23.142Z } - match: { hits.hits.2._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.3._id: AQAAALB9sSWsyrL5AAABeRnRZ1M } + - match: { hits.hits.3._id: cZZNs7B9sSWsyrL5AAABeRnRZ1M } - match: { hits.hits.3._source.@timestamp: 2021-04-28T18:50:24.467Z } - match: { hits.hits.3._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.4._id: AQAAALB9sSWsyrL5AAABeRnRtXM } + - match: { hits.hits.4._id: cZZNs7B9sSWsyrL5AAABeRnRtXM } - match: { hits.hits.4._source.@timestamp: 2021-04-28T18:50:44.467Z } - match: { hits.hits.4._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.5._id: AQAAAMfoxSs_KdA5AAABeRnR11Y } + - match: { hits.hits.5._id: cn4excfoxSs_KdA5AAABeRnR11Y } - match: { hits.hits.5._source.@timestamp: 2021-04-28T18:50:53.142Z } - match: { hits.hits.5._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.6._id: AQAAAMfoxSs_KdA5AAABeRnR_mY } + - match: { hits.hits.6._id: cn4excfoxSs_KdA5AAABeRnR_mY } - match: { hits.hits.6._source.@timestamp: 2021-04-28T18:51:03.142Z } - match: { hits.hits.6._source.k8s.pod.uid: df3145b3-0563-4d3b-a0f7-897eb2876ea9 } - - match: { hits.hits.7._id: AQAAALB9sSWsyrL5AAABeRnSA5M } + - match: { hits.hits.7._id: cZZNs7B9sSWsyrL5AAABeRnSA5M } - match: { hits.hits.7._source.@timestamp: 2021-04-28T18:51:04.467Z } - match: { hits.hits.7._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } - - match: { hits.hits.8._id: AQAAALB9sSWsyrL5AAABeRnS7fM } + - match: { hits.hits.8._id: cZZNs7B9sSWsyrL5AAABeRnS7fM } - match: { hits.hits.8._source.@timestamp: 2021-04-28T18:52:04.467Z } - match: { hits.hits.8._source.k8s.pod.uid: 947e4ced-1786-4e53-9e0c-5c447e959507 } @@ -170,7 +170,7 @@ index a new document on top of an old one: network: tx: 111434595272 rx: 430605511 - - match: {_id: AQAAAMfoxSs_KdA5AAABeRnR_mY} + - match: {_id: cn4excfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -215,7 +215,7 @@ index a new document on top of an old one over bulk: body: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: {items.0.index._id: AQAAAMfoxSs_KdA5AAABeRnR_mY} + - match: {items.0.index._id: cn4excfoxSs_KdA5AAABeRnR_mY} - do: search: @@ -239,7 +239,7 @@ create operation on top of old document fails: reason: id generation changed in 8.2 - do: - catch: "/\\[AQAAAMfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" + catch: "/\\[cn4excfoxSs_KdA5AAABeRnR_mY\\]\\[.*@2021-04-28T18:51:03.142Z\\]: version conflict, document already exists \\(current version \\[1\\]\\)/" index: refresh: true index: test @@ -268,7 +268,7 @@ create operation on top of old document fails over bulk: body: - '{"create": {}}' - '{"@timestamp": "2021-04-28T18:51:03.142Z", "metricset": "pod", "k8s": {"pod": {"name": "dog", "uid":"df3145b3-0563-4d3b-a0f7-897eb2876ea9", "ip": "10.10.55.3", "network": {"tx": 111434595272, "rx": 430605511}}}}' - - match: { items.0.create.error.reason: "[AQAAAMfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } + - match: { items.0.create.error.reason: "[cn4excfoxSs_KdA5AAABeRnR_mY][KCjEJ9R_BgO8TRX2QOd6dpQ5ihHD--qoyLTiOy0pmP6_RAIE-e0-dKQ@2021-04-28T18:51:03.142Z]: version conflict, document already exists (current version [1])" } --- ids query: @@ -284,12 +284,12 @@ ids query: - field: k8s.pod.network.tx query: ids: - values: ["AQAAAMfoxSs_KdA5AAABeRnR11Y", "AQAAAMfoxSs_KdA5AAABeRnR_mY"] + values: ["cn4excfoxSs_KdA5AAABeRnR11Y", "cn4excfoxSs_KdA5AAABeRnR_mY"] sort: ["@timestamp"] - match: {hits.total.value: 2} - - match: {hits.hits.0._id: "AQAAAMfoxSs_KdA5AAABeRnR11Y"} + - match: {hits.hits.0._id: "cn4excfoxSs_KdA5AAABeRnR11Y"} - match: {hits.hits.0.fields.k8s\.pod\.network\.tx: [1434587694]} - - match: {hits.hits.1._id: "AQAAAMfoxSs_KdA5AAABeRnR_mY" } + - match: {hits.hits.1._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } - match: {hits.hits.1.fields.k8s\.pod\.network\.tx: [1434595272]} --- @@ -301,9 +301,9 @@ get: - do: get: index: test - id: AQAAALB9sSWsyrL5AAABeRnSA5M + id: cZZNs7B9sSWsyrL5AAABeRnSA5M - match: {_index: test} - - match: {_id: AQAAALB9sSWsyrL5AAABeRnSA5M} + - match: {_id: cZZNs7B9sSWsyrL5AAABeRnSA5M} - match: _source: "@timestamp": "2021-04-28T18:51:04.467Z" @@ -351,7 +351,7 @@ delete: - do: delete: index: test - id: AQAAAMfoxSs_KdA5AAABeRnR_mY + id: cn4excfoxSs_KdA5AAABeRnR_mY - match: {result: deleted} --- @@ -391,20 +391,20 @@ delete over _bulk: mget: index: test body: - ids: [ AQAAAMfoxSs_KdA5AAABeRnR_mY, AQAAAMfoxSs_KdA5AAABeRnR11Y ] + ids: [ cn4excfoxSs_KdA5AAABeRnR_mY, cn4excfoxSs_KdA5AAABeRnR11Y ] - match: { docs.0._index: "test" } - - match: { docs.0._id: "AQAAAMfoxSs_KdA5AAABeRnR_mY" } + - match: { docs.0._id: "cn4excfoxSs_KdA5AAABeRnR_mY" } - match: { docs.0.found: true } - match: { docs.1._index: "test" } - - match: { docs.1._id: "AQAAAMfoxSs_KdA5AAABeRnR11Y" } + - match: { docs.1._id: "cn4excfoxSs_KdA5AAABeRnR11Y" } - match: { docs.1.found: true } - do: bulk: index: test body: - - '{"delete": {"_id": "AQAAAMfoxSs_KdA5AAABeRnR_mY"}}' - - '{"delete": {"_id": "AQAAAMfoxSs_KdA5AAABeRnR11Y"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR_mY"}}' + - '{"delete": {"_id": "cn4excfoxSs_KdA5AAABeRnR11Y"}}' - '{"delete": {"_id": "not found ++ not found"}}' - match: {items.0.delete.result: deleted} - match: {items.1.delete.result: deleted} @@ -454,7 +454,7 @@ routing_path matches deep object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"bar": {"baz": {"uid": "uid1"}}}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: AAAAAMhJrjH1fFX8AAABeRnRGTM} + - match: {items.0.index._id: OcEOGchJrjH1fFX8AAABeRnRGTM} --- routing_path matches object: @@ -495,4 +495,4 @@ routing_path matches object: - '{"index": {}}' - '{"@timestamp": "2021-04-28T18:50:04.467Z", "dim": {"foo": {"uid": "uid1"}}}' - match: {items.0.index.result: created} - - match: {items.0.index._id: AAAAAG9JKwAyp1bZAAABeRnRGTM} + - match: {items.0.index._id: 8bgiqW9JKwAyp1bZAAABeRnRGTM} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java index 833283408ee6f..80ed275fdc87e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java @@ -93,7 +93,7 @@ public void postParse(DocumentParserContext context) { if (context.indexSettings().getMode() == IndexMode.TIME_SERIES && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { var field = new SortedDocValuesField(NAME, Uid.encodeId(context.sourceToParse().id())); - context.doc().add(field); + context.rootDoc().add(field); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 8af74e03f8605..3360ab86ff01b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -47,7 +47,7 @@ public void testSimpleRoutingRejectsEmptyId() { IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() ); IndexRequest req = new IndexRequest().id(""); - Exception e = expectThrows(IllegalArgumentException.class, () -> indexRouting.process(req)); + Exception e = expectThrows(IllegalArgumentException.class, () -> indexRouting.preProcess(req)); assertThat(e.getMessage(), equalTo("if _id is specified it must not be empty")); } @@ -57,7 +57,7 @@ public void testSimpleRoutingAcceptsId() { ); String id = randomAlphaOfLength(10); IndexRequest req = new IndexRequest().id(id); - indexRouting.process(req); + indexRouting.preProcess(req); assertThat(req.id(), equalTo(id)); assertThat(req.getAutoGeneratedTimestamp(), equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP)); } @@ -67,7 +67,7 @@ public void testSimpleRoutingAssignedRandomId() { IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() ); IndexRequest req = new IndexRequest(); - indexRouting.process(req); + indexRouting.preProcess(req); assertThat(req.id(), not(nullValue())); assertThat(req.getAutoGeneratedTimestamp(), not(equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))); } @@ -469,7 +469,7 @@ public void testRoutingAllowsId() { IndexRouting indexRouting = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); String id = randomAlphaOfLength(5); IndexRequest req = new IndexRequest().id(id); - indexRouting.process(req); + indexRouting.preProcess(req); assertThat(req.id(), equalTo(id)); } @@ -482,7 +482,7 @@ public void testRoutingAllowsId() { public void testRoutingPathLeavesIdNull() { IndexRouting indexRouting = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); IndexRequest req = new IndexRequest(); - indexRouting.process(req); + indexRouting.preProcess(req); assertThat(req.id(), nullValue()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 5b49ed3446f65..524d862ca5eee 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -684,37 +684,30 @@ public static Iterable params() { private final TestCase testCase; + private static final int ROUTING_HASH = 7; + public TsidExtractingIdFieldMapperTests(@Named("testCase") TestCase testCase) { this.testCase = testCase; } public void testExpectedId() throws IOException { - assertThat(parse(null, mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); + assertThat( + parse(TimeSeriesRoutingIdFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.source).id(), + equalTo(testCase.expectedId) + ); } public void testProvideExpectedId() throws IOException { assertThat(parse(testCase.expectedId, mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); } - public void testProvideWrongId() { - String wrongId = testCase.expectedId + "wrong"; - Exception e = expectThrows(DocumentParsingException.class, () -> parse(wrongId, mapperService(), testCase.source)); - assertThat( - e.getCause().getMessage(), - equalTo( - "_id must be unset or set to [" - + testCase.expectedId - + "] but was [" - + testCase.expectedId - + "wrong] because [index] is in time_series mode" - ) - ); - } - public void testEquivalentSources() throws IOException { MapperService mapperService = mapperService(); for (CheckedConsumer equivalent : testCase.equivalentSources) { - assertThat(parse(null, mapperService, equivalent).id(), equalTo(testCase.expectedId)); + assertThat( + parse(TimeSeriesRoutingIdFieldMapper.encode(ROUTING_HASH), mapperService, equivalent).id(), + equalTo(testCase.expectedId) + ); } } @@ -736,15 +729,6 @@ public void testRoutingPathCompliant() throws IOException { assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(SHARD_ID)); } - private int indexShard(IndexRouting indexRouting) throws IOException { - try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { - builder.startObject(); - testCase.source.accept(builder); - builder.endObject(); - return indexRouting.indexShard(null, null, builder.contentType(), BytesReference.bytes(builder)); - } - } - private Settings indexSettings(IndexVersion version) { return Settings.builder() .put(IndexSettings.MODE.getKey(), "time_series") @@ -803,7 +787,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { public void testSourceDescription() throws IOException { assertThat(TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext()), equalTo("a time series document")); - ParsedDocument d = parse(null, mapperService(), testCase.randomSource()); + ParsedDocument d = parse(TimeSeriesRoutingIdFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.randomSource()); IndexableField timestamp = d.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(timestamp)), @@ -833,7 +817,9 @@ private TestDocumentParserContext documentParserContext(IndexableField... fields public void testParsedDescription() throws IOException { assertThat( - TsidExtractingIdFieldMapper.INSTANCE.documentDescription(parse(null, mapperService(), testCase.randomSource())), + TsidExtractingIdFieldMapper.INSTANCE.documentDescription( + parse(TimeSeriesRoutingIdFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.randomSource()) + ), equalTo("[" + testCase.expectedId + "][" + testCase.expectedTsid + "@" + testCase.expectedTimestamp + "]") ); } From c13e0fff6bf1df755e2c4e1c2b3c7fc0f73aecfe Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 15 Feb 2024 17:02:46 +0200 Subject: [PATCH 25/51] delete unused yaml --- docs/changelog/105073.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/105073.yaml diff --git a/docs/changelog/105073.yaml b/docs/changelog/105073.yaml deleted file mode 100644 index 62b22f17cf4ea..0000000000000 --- a/docs/changelog/105073.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 105073 -summary: Support non-keyword fields as dimension subfields of pass-through object -area: TSDB -type: enhancement -issues: [] From 1f186037bce13e0953f8eb7d599686d665965dd0 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 19 Feb 2024 09:00:00 +0200 Subject: [PATCH 26/51] submit test fixes --- .../index/mapper/DocumentMapperTests.java | 2 + .../index/mapper/IdLoaderTests.java | 44 +++++++++++-------- .../indices/IndicesModuleTests.java | 2 + .../recovery/RecoverySourceHandlerTests.java | 3 +- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 144bfa3e8701e..7d521f0aea22e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -325,6 +325,7 @@ public void testEmptyDocumentMapper() { .item(RoutingFieldMapper.class) .item(SeqNoFieldMapper.class) .item(SourceFieldMapper.class) + .item(TimeSeriesRoutingIdFieldMapper.class) .item(VersionFieldMapper.class) ); List matching = new ArrayList<>(documentMapper.mappers().getMatchingFieldNames("*")); @@ -339,6 +340,7 @@ public void testEmptyDocumentMapper() { .item(IndexFieldMapper.CONTENT_TYPE) .item(NestedPathFieldMapper.NAME) .item(RoutingFieldMapper.CONTENT_TYPE) + .item(TimeSeriesRoutingIdFieldMapper.NAME) .item(SeqNoFieldMapper.CONTENT_TYPE) .item(SourceFieldMapper.CONTENT_TYPE) .item(VersionFieldMapper.CONTENT_TYPE) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index 9afde7c7e2b1f..fb5004307d48f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -46,6 +46,8 @@ public class IdLoaderTests extends ESTestCase { + private final int routingHash = randomInt(); + public void testSynthesizeIdSimple() throws Exception { var idLoader = IdLoader.createTsIdLoader(null, null); @@ -61,11 +63,11 @@ public void testSynthesizeIdSimple() throws Exception { assertThat(leafReader.numDocs(), equalTo(3)); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 1, 2 }); // NOTE: time series data is ordered by (tsid, timestamp) - assertThat(leaf.getId(0), equalTo(expectedId(docs.get(2)))); - assertThat(leaf.getId(1), equalTo(expectedId(docs.get(0)))); - assertThat(leaf.getId(2), equalTo(expectedId(docs.get(1)))); + assertThat(leaf.getId(0), equalTo(expectedId(docs.get(2), routingHash))); + assertThat(leaf.getId(1), equalTo(expectedId(docs.get(0), routingHash))); + assertThat(leaf.getId(2), equalTo(expectedId(docs.get(1), routingHash))); }; - prepareIndexReader(indexAndForceMerge(docs), verify, false); + prepareIndexReader(indexAndForceMerge(docs, routingHash), verify, false); } public void testSynthesizeIdMultipleSegments() throws Exception { @@ -94,15 +96,15 @@ public void testSynthesizeIdMultipleSegments() throws Exception { ); CheckedConsumer buildIndex = writer -> { for (Doc doc : docs1) { - indexDoc(writer, doc); + indexDoc(writer, doc, routingHash); } writer.flush(); for (Doc doc : docs2) { - indexDoc(writer, doc); + indexDoc(writer, doc, routingHash); } writer.flush(); for (Doc doc : docs3) { - indexDoc(writer, doc); + indexDoc(writer, doc, routingHash); } writer.flush(); }; @@ -113,22 +115,22 @@ public void testSynthesizeIdMultipleSegments() throws Exception { assertThat(leafReader.numDocs(), equalTo(docs1.size())); var leaf = idLoader.leaf(null, leafReader, IntStream.range(0, docs1.size()).toArray()); for (int i = 0; i < docs1.size(); i++) { - assertThat(leaf.getId(i), equalTo(expectedId(docs1.get(i)))); + assertThat(leaf.getId(i), equalTo(expectedId(docs1.get(i), routingHash))); } } { LeafReader leafReader = indexReader.leaves().get(1).reader(); assertThat(leafReader.numDocs(), equalTo(docs2.size())); var leaf = idLoader.leaf(null, leafReader, new int[] { 0, 3 }); - assertThat(leaf.getId(0), equalTo(expectedId(docs2.get(0)))); - assertThat(leaf.getId(3), equalTo(expectedId(docs2.get(3)))); + assertThat(leaf.getId(0), equalTo(expectedId(docs2.get(0), routingHash))); + assertThat(leaf.getId(3), equalTo(expectedId(docs2.get(3), routingHash))); } { LeafReader leafReader = indexReader.leaves().get(2).reader(); assertThat(leafReader.numDocs(), equalTo(docs3.size())); var leaf = idLoader.leaf(null, leafReader, new int[] { 1, 2 }); - assertThat(leaf.getId(1), equalTo(expectedId(docs3.get(1)))); - assertThat(leaf.getId(2), equalTo(expectedId(docs3.get(2)))); + assertThat(leaf.getId(1), equalTo(expectedId(docs3.get(1), routingHash))); + assertThat(leaf.getId(2), equalTo(expectedId(docs3.get(2), routingHash))); } { LeafReader leafReader = indexReader.leaves().get(2).reader(); @@ -168,7 +170,7 @@ public void testSynthesizeIdRandom() throws Exception { for (int j = 0; j < numberOfSamples; j++) { Doc doc = new Doc(startTime++, dimensions); randomDocs.add(doc); - expectedIDs.add(expectedId(doc)); + expectedIDs.add(expectedId(doc, routingHash)); } } CheckedConsumer verify = indexReader -> { @@ -181,14 +183,14 @@ public void testSynthesizeIdRandom() throws Exception { assertTrue("docId=" + i + " id=" + actualId, expectedIDs.remove(actualId)); } }; - prepareIndexReader(indexAndForceMerge(randomDocs), verify, false); + prepareIndexReader(indexAndForceMerge(randomDocs, routingHash), verify, false); assertThat(expectedIDs, empty()); } - private static CheckedConsumer indexAndForceMerge(List docs) { + private static CheckedConsumer indexAndForceMerge(List docs, int routingHash) { return writer -> { for (Doc doc : docs) { - indexDoc(writer, doc); + indexDoc(writer, doc, routingHash); } writer.forceMerge(1); }; @@ -207,6 +209,7 @@ private void prepareIndexReader( } Sort sort = new Sort( new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false), + new SortField(TimeSeriesRoutingIdFieldMapper.NAME, SortField.Type.STRING, false), new SortedNumericSortField(DataStreamTimestampFieldMapper.DEFAULT_PATH, SortField.Type.LONG, true) ); config.setIndexSort(sort); @@ -220,7 +223,7 @@ private void prepareIndexReader( } } - private static void indexDoc(IndexWriter iw, Doc doc) throws IOException { + private static void indexDoc(IndexWriter iw, Doc doc, int routingHash) throws IOException { final TimeSeriesIdFieldMapper.TimeSeriesIdBuilder builder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); final List fields = new ArrayList<>(); @@ -237,10 +240,13 @@ private static void indexDoc(IndexWriter iw, Doc doc) throws IOException { } BytesRef tsid = builder.buildTsidHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); + fields.add( + new SortedDocValuesField(TimeSeriesRoutingIdFieldMapper.NAME, Uid.encodeId(TimeSeriesRoutingIdFieldMapper.encode(routingHash))) + ); iw.addDocument(fields); } - private static String expectedId(Doc doc) throws IOException { + private static String expectedId(Doc doc, int routingHash) throws IOException { var timeSeriesIdBuilder = new TimeSeriesIdFieldMapper.TimeSeriesIdBuilder(null); for (Dimension dimension : doc.dimensions) { if (dimension.value instanceof Number n) { @@ -249,7 +255,7 @@ private static String expectedId(Doc doc) throws IOException { timeSeriesIdBuilder.addString(dimension.field, dimension.value.toString()); } } - return TsidExtractingIdFieldMapper.createId(randomInt(), timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp); + return TsidExtractingIdFieldMapper.createId(routingHash, timeSeriesIdBuilder.buildTsidHash().toBytesRef(), doc.timestamp); } private static IndexRouting.ExtractFromSource createRouting(List routingPaths) { diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index 1648e38a3f0b9..ea35d1708abb4 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; @@ -80,6 +81,7 @@ public Map getMetadataMappers() { IdFieldMapper.NAME, RoutingFieldMapper.NAME, TimeSeriesIdFieldMapper.NAME, + TimeSeriesRoutingIdFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, NestedPathFieldMapper.NAME, diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 82fb694db6c66..3a4e1daa1958d 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -56,6 +56,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLease; import org.elasticsearch.index.seqno.RetentionLeases; @@ -517,7 +518,7 @@ private TimeSeriesModeIndexOpFactory() throws IOException { @Override public Engine.Index createIndexOp(int docIdent) { - SourceToParse source = new SourceToParse(null, new BytesArray(Strings.format(""" + SourceToParse source = new SourceToParse(TimeSeriesRoutingIdFieldMapper.encode(0), new BytesArray(Strings.format(""" { "@timestamp": %s, "dim": "dim" From 72cd22504550db9ac85f3217c7b009f69faa56cb Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 19 Feb 2024 10:15:42 +0200 Subject: [PATCH 27/51] sync --- .../index/mapper/TsidExtractingIdFieldMapper.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index c980fbcce93f0..2d0f958c55585 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; +import java.util.Base64; import java.util.Locale; /** @@ -91,6 +92,18 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); } + public static String createId(int routingId, BytesRef tsid, long timestamp) { + Hash128 hash = new Hash128(); + MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); + + byte[] bytes = new byte[20]; + ByteUtils.writeIntLE(routingId, bytes, 0); + ByteUtils.writeLongLE(hash.h1, bytes, 4); + ByteUtils.writeLongBE(timestamp, bytes, 12); // Big Ending shrinks the inverted index by ~37% + + return Base64.getUrlEncoder().withoutPadding().encodeToString(bytes); + } + public static String createId( boolean dynamicMappersExists, IndexRouting.ExtractFromSource.Builder routingBuilder, From 30b4dbbca62b03c9466dce88230fee4c0b86f7d4 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 19 Feb 2024 10:29:13 +0200 Subject: [PATCH 28/51] restrict id override to latest version --- .../java/org/elasticsearch/cluster/routing/IndexRouting.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 5259dc9ebb209..566a537ee15c2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentParser; @@ -240,6 +241,7 @@ public void collectSearchShards(String routing, IntConsumer consumer) { public static class ExtractFromSource extends IndexRouting { private final Predicate isRoutingPath; private final XContentParserConfiguration parserConfig; + private final boolean indexTracksRoutingId; private int routingId = 0; @@ -248,6 +250,7 @@ public static class ExtractFromSource extends IndexRouting { if (metadata.isRoutingPartitionedIndex()) { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } + indexTracksRoutingId = metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID); List routingPaths = metadata.getRoutingPaths(); isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new)); this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); @@ -255,7 +258,7 @@ public static class ExtractFromSource extends IndexRouting { @Override public void postProcess(IndexRequest indexRequest) { - if (indexRequest.id() == null) { + if (indexRequest.id() == null && indexTracksRoutingId) { indexRequest.id(TimeSeriesRoutingIdFieldMapper.encode(routingId)); } } From 8cd545658e048046a40bb46b05b4135757bdc863 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 19 Feb 2024 18:13:44 +0200 Subject: [PATCH 29/51] update painless path --- .../painless/action/PainlessExecuteAction.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 6048020f09fbf..8c49fa1240ca1 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -56,6 +56,7 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.OnScriptError; @@ -811,7 +812,13 @@ private static Response prepareRamIndex( XContentType xContentType = request.contextSetup.xContentType; String id; if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - id = TimeSeriesRoutingIdFieldMapper.encode(0); + if (indexService.getIndexSettings() + .getIndexVersionCreated() + .onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { + id = TimeSeriesRoutingIdFieldMapper.encode(0); + } else { + id = null; + } } else { id = "_id"; } From 302701bb31a1e42f8716e38e5f6ce097bb0e996a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 4 Mar 2024 16:28:16 +0200 Subject: [PATCH 30/51] test fix --- .../org/elasticsearch/index/codec/PerFieldMapperCodec.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java index 2a2ae7245d996..ba7d4c034044f 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java @@ -115,8 +115,8 @@ boolean useTSDBDocValuesFormat(final String field) { private boolean excludeFields(String fieldName) { // Avoid using tsdb codec for fields like _seq_no, _primary_term. - // But _tsid should always use the tesbd codec. - return fieldName.startsWith("_") && fieldName.equals("_tsid") == false; + // But _tsid and _routing_path should always use the tsdb codec. + return fieldName.startsWith("_") && fieldName.equals("_tsid") == false && fieldName.equals("_routing_id") == false; } private boolean isTimeSeriesModeIndex() { From dc0a8872ac008f79ad78fcee02fc95e17d0e7e84 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 4 Mar 2024 22:10:43 +0200 Subject: [PATCH 31/51] check for invalid id --- .../elasticsearch/index/IndexVersions.java | 2 +- .../TimeSeriesRoutingIdFieldMapper.java | 3 +- .../mapper/TsidExtractingIdFieldMapper.java | 8 +- .../mapper/TimeSeriesIdFieldMapperTests.java | 117 +++++++++++------- 4 files changed, 83 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 34020f40c2c89..8428d20a2b96e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -102,7 +102,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); public static final IndexVersion UPGRADE_TO_LUCENE_9_10 = def(8_503_00_0, Version.LUCENE_9_10_0); - public static final IndexVersion TIME_SERIES_ROUTING_ID_IN_ID = def(8_504_00_0, Version.LUCENE_9_10_2); + public static final IndexVersion TIME_SERIES_ROUTING_ID_IN_ID = def(8_504_00_0, Version.LUCENE_9_10_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java index 80ed275fdc87e..e2a6cb607b6cc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java @@ -91,7 +91,8 @@ private TimeSeriesRoutingIdFieldMapper() { @Override public void postParse(DocumentParserContext context) { if (context.indexSettings().getMode() == IndexMode.TIME_SERIES - && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { + && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID) + && context.sourceToParse().id() != null) { var field = new SortedDocValuesField(NAME, Uid.encodeId(context.sourceToParse().id())); context.rootDoc().add(field); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 6f23af924cc92..5d874d75ad9f0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -82,9 +82,15 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra ) ); } - } else { + } else if (context.sourceToParse().id() != null) { int routingId = TimeSeriesRoutingIdFieldMapper.decode(context.sourceToParse().id()); id = createId(routingId, tsid, timestamp); + } else { + throw new IllegalArgumentException( + "_id was null but must be set because index [" + + context.indexSettings().getIndexMetadata().getIndex().getName() + + "] is in time_series mode" + ); } context.id(id); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 94a0f2296bbfb..4f79847e9b15a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -666,16 +666,44 @@ public void testParseWithDynamicMapping() { .put(IndexSettings.MODE.getKey(), "time_series") .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") .build(); - // without _id - { - MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); - SourceToParse source = new SourceToParse(null, new BytesArray(""" - { - "@timestamp": 1609459200000, - "dim": "6a841a21", - "value": 100 - }"""), XContentType.JSON); - Engine.Index index = IndexShard.prepareIndex( + MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); + SourceToParse source = new SourceToParse(TimeSeriesRoutingIdFieldMapper.encode(0), new BytesArray(""" + { + "@timestamp": 1609459200000, + "dim": "6a841a21", + "value": 100 + }"""), XContentType.JSON); + Engine.Index index = IndexShard.prepareIndex( + mapper, + source, + UNASSIGNED_SEQ_NO, + randomNonNegativeLong(), + Versions.MATCH_ANY, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + -1, + false, + UNASSIGNED_SEQ_NO, + 0, + System.nanoTime() + ); + assertNotNull(index.parsedDoc().dynamicMappingsUpdate()); + } + + public void testParseWithDynamicMappingInvalidId() { + Settings indexSettings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") + .build(); + MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); + SourceToParse source = new SourceToParse("no-such-id-exists", new BytesArray(""" + { + "@timestamp": 1609459200000, + "dim": "6a841a21", + "value": 100 + }"""), XContentType.JSON); + var failure = expectThrows(DocumentParsingException.class, () -> { + IndexShard.prepareIndex( mapper, source, UNASSIGNED_SEQ_NO, @@ -689,40 +717,41 @@ public void testParseWithDynamicMapping() { 0, System.nanoTime() ); - assertNotNull(index.parsedDoc().dynamicMappingsUpdate()); - } - // with _id - { - MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); - SourceToParse source = new SourceToParse("no-such-tsid", new BytesArray(""" - { - "@timestamp": 1609459200000, - "dim": "6a841a21", - "value": 100 - }"""), XContentType.JSON); - var failure = expectThrows(DocumentParsingException.class, () -> { - IndexShard.prepareIndex( - mapper, - source, - UNASSIGNED_SEQ_NO, - randomNonNegativeLong(), - Versions.MATCH_ANY, - VersionType.INTERNAL, - Engine.Operation.Origin.PRIMARY, - -1, - false, - UNASSIGNED_SEQ_NO, - 0, - System.nanoTime() - ); - }); - assertThat( - failure.getMessage(), - equalTo( - "[5:1] failed to parse: _id must be unset or set to [AAAAAMpxfIC8Wpr0AAABdrs-cAA]" - + " but was [no-such-tsid] because [index] is in time_series mode" - ) + }); + assertThat(failure.getMessage(), equalTo("[5:1] failed to parse: Last unit does not have enough valid bits")); + } + + public void testParseWithDynamicMappingNullId() { + Settings indexSettings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") + .build(); + MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); + SourceToParse source = new SourceToParse(null, new BytesArray(""" + { + "@timestamp": 1609459200000, + "dim": "6a841a21", + "value": 100 + }"""), XContentType.JSON); + var failure = expectThrows(DocumentParsingException.class, () -> { + IndexShard.prepareIndex( + mapper, + source, + UNASSIGNED_SEQ_NO, + randomNonNegativeLong(), + Versions.MATCH_ANY, + VersionType.INTERNAL, + Engine.Operation.Origin.PRIMARY, + -1, + false, + UNASSIGNED_SEQ_NO, + 0, + System.nanoTime() ); - } + }); + assertThat( + failure.getMessage(), + equalTo("[5:1] failed to parse: _id was null but must be set because index [index] is in time_series mode") + ); } } From f14721ecdd4edfd99181689fbf6d3fc2a3e91d86 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 4 Mar 2024 22:30:45 +0200 Subject: [PATCH 32/51] fix painless tests --- .../rest-api-spec/test/update_by_query/90_tsdb.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml index cee4395eb87f8..2c31847391cc2 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml @@ -77,7 +77,7 @@ update tag field: update dimension field: # TODO better error message - do: - catch: bad_request + catch: conflict update_by_query: index: tsdb body: @@ -86,17 +86,17 @@ update dimension field: source: ctx._source.k8s.pod.uid = "12342134" - match: {updated: 0} - - match: {version_conflicts: 0} + - match: {version_conflicts: 1} - match: {batches: 1} - match: {throttled_millis: 0} - gte: { took: 0 } - - match: {failures.0.cause.caused_by.reason: /_id\ must\ be\ unset\ or\ set\ to\ .+/} + - match: {failures.0.cause.reason: /version\ conflict/} --- update timestamp: # TODO better error message - do: - catch: bad_request + catch: conflict update_by_query: index: tsdb body: @@ -105,8 +105,8 @@ update timestamp: source: ctx._source["@timestamp"] = "2021-04-28T18:50:33.142Z" - match: {updated: 0} - - match: {version_conflicts: 0} + - match: {version_conflicts: 1} - match: {batches: 1} - match: {throttled_millis: 0} - gte: { took: 0 } - - match: {failures.0.cause.caused_by.reason: /_id\ must\ be\ unset\ or\ set\ to\ .+/} + - match: {failures.0.cause.reason: /version\ conflict/} From c7a9ebc11b46c78d68766dfa567a7652832680db Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Mon, 4 Mar 2024 22:52:35 +0200 Subject: [PATCH 33/51] restore bwc routing changes --- .../cluster/routing/IndexRouting.java | 4 ++++ .../search/DefaultSearchContext.java | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 566a537ee15c2..234d34a1e8514 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -256,6 +256,10 @@ public static class ExtractFromSource extends IndexRouting { this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); } + public boolean matchesField(String fieldName) { + return isRoutingPath.test(fieldName); + } + @Override public void postProcess(IndexRequest indexRequest) { if (indexRequest.id() == null && indexTracksRoutingId) { diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 4fac628694fcd..0bf4fd1b951a2 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -34,7 +34,9 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexOrdinalsFieldData; import org.elasticsearch.index.mapper.IdLoader; +import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceLoader; import org.elasticsearch.index.query.AbstractQueryBuilder; @@ -75,6 +77,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.Executor; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.LongSupplier; @@ -899,6 +903,20 @@ public IdLoader newIdLoader() { if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); routingPaths = indexService.getMetadata().getRoutingPaths(); + for (String routingField : routingPaths) { + if (routingField.contains("*")) { + // In case the routing fields include path matches, find any matches and add them as distinct fields + // to the routing path. + Set matchingRoutingPaths = new TreeSet<>(routingPaths); + for (Mapper mapper : indexService.mapperService().mappingLookup().fieldMappers()) { + if (mapper instanceof KeywordFieldMapper && indexRouting.matchesField(mapper.name())) { + matchingRoutingPaths.add(mapper.name()); + } + } + routingPaths = new ArrayList<>(matchingRoutingPaths); + break; + } + } } return IdLoader.createTsIdLoader(indexRouting, routingPaths); } else { From 4b1d53294a4e3e8798afd84269342a19497ef4d6 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 5 Mar 2024 17:18:22 +0200 Subject: [PATCH 34/51] add unittest --- docs/changelog/105501.yaml | 3 +- .../test/nodes.stats/11_indices_metrics.yml | 33 +++--- .../test/tsdb/140_routing_path.yml | 4 +- .../rest-api-spec/test/tsdb/20_mapping.yml | 8 +- .../org/elasticsearch/index/IndexMode.java | 19 ++++ .../TimeSeriesRoutingIdFieldMapper.java | 2 +- .../index/mapper/DocumentMapperTests.java | 2 - .../mapper/FieldFilterMapperPluginTests.java | 3 +- .../TimeSeriesRoutingIdFieldMapperTests.java | 105 ++++++++++++++++++ 9 files changed, 150 insertions(+), 29 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapperTests.java diff --git a/docs/changelog/105501.yaml b/docs/changelog/105501.yaml index 49691bb5d82f1..1bf658529af58 100644 --- a/docs/changelog/105501.yaml +++ b/docs/changelog/105501.yaml @@ -1,6 +1,5 @@ pr: 105501 -summary: Support non-keyword fields as dimension subfields of pass-through object - - with `TimeSeriesRoutingIdFieldMapper` +summary: Introduce TimeSeriesRoutingIdFieldMapper to support non-keyword dimensions area: TSDB type: enhancement issues: [] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml index 3536c20071ab7..b119a1a1d94f3 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml @@ -483,29 +483,28 @@ # 11. _source # 12. _tier # 13. _version - # 14. _routing_id - # 15. @timestamp - # 16. authors.age - # 17. authors.company - # 18. authors.company.keyword - # 19. authors.name.last_name - # 20. authors.name.first_name - # 21. authors.name.full_name - # 22. link - # 23. title - # 24. url + # 14. @timestamp + # 15. authors.age + # 16. authors.company + # 17. authors.company.keyword + # 18. authors.name.last_name + # 19. authors.name.first_name + # 20. authors.name.full_name + # 21. link + # 22. title + # 23. url # Object mappers: - # 25. authors - # 26. authors.name + # 24. authors + # 25. authors.name # Runtime field mappers: - # 27. a_source_field + # 26. a_source_field - gte: { nodes.$node_id.indices.mappings.total_count: 26 } - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead - - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 27648 } - - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 27 } + - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 } + - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 26 } - is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead - - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 27648 } + - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 26624 } --- "indices mappings does not exist in shards level": diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml index 53a1821ee8351..2ee2391458b03 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml @@ -90,8 +90,8 @@ missing routing path field: --- missing dimension on routing path field: - skip: - version: " - 8.12.99" - reason: error message changed in 8.13.0 + version: " - 8.13.99" + reason: error message changed in 8.14.0 - do: catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[tag\] was not a dimension./' diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index c823984450c3c..8ccac3f95a992 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -121,8 +121,8 @@ top level wildcard dim object: --- exact match object type: - skip: - version: " - 8.12.99" - reason: routing_path error message updated in 8.13.0 + version: " - 8.13.99" + reason: routing_path error message updated in 8.14.0 - do: catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[dim\] was \[object\]./' @@ -154,8 +154,8 @@ exact match object type: --- non keyword matches routing_path: - skip: - version: " - 8.12.99" - reason: routing_path error message updated in 8.13.0 + version: " - 8.13.99" + reason: routing_path error message updated in 8.14.0 - do: catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./' diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 05afc14e0f0cd..969b5209daa9e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -28,6 +28,7 @@ import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import java.io.IOException; @@ -92,6 +93,12 @@ public MetadataFieldMapper timeSeriesIdFieldMapper() { return null; } + @Override + public MetadataFieldMapper timeSeriesRoutingIdFieldMapper() { + // non time-series indices must not have a TimeSeriesRoutingIdFieldMapper + return null; + } + @Override public IdFieldMapper idFieldMapperWithoutFieldData() { return ProvidedIdFieldMapper.NO_FIELD_DATA; @@ -185,6 +192,11 @@ public MetadataFieldMapper timeSeriesIdFieldMapper() { return TimeSeriesIdFieldMapper.INSTANCE; } + @Override + public MetadataFieldMapper timeSeriesRoutingIdFieldMapper() { + return TimeSeriesRoutingIdFieldMapper.INSTANCE; + } + public IdFieldMapper idFieldMapperWithoutFieldData() { return TsidExtractingIdFieldMapper.INSTANCE; } @@ -322,6 +334,13 @@ public String getName() { */ public abstract MetadataFieldMapper timeSeriesIdFieldMapper(); + /** + * Return an instance of the {@link TimeSeriesIdFieldMapper} that generates + * the _tsid field. The field mapper will be added to the list of the metadata + * field mappers for the index. + */ + public abstract MetadataFieldMapper timeSeriesRoutingIdFieldMapper(); + /** * How {@code time_series_dimension} fields are handled by indices in this mode. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java index e2a6cb607b6cc..a223d9ac3cfa6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java @@ -45,7 +45,7 @@ public class TimeSeriesRoutingIdFieldMapper extends MetadataFieldMapper { public static final TimeSeriesRoutingIdFieldMapper INSTANCE = new TimeSeriesRoutingIdFieldMapper(); - public static final TypeParser PARSER = new FixedTypeParser(c -> INSTANCE); + public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesRoutingIdFieldMapper()); static final class TimeSeriesRoutingIdFieldType extends MappedFieldType { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 7d521f0aea22e..144bfa3e8701e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -325,7 +325,6 @@ public void testEmptyDocumentMapper() { .item(RoutingFieldMapper.class) .item(SeqNoFieldMapper.class) .item(SourceFieldMapper.class) - .item(TimeSeriesRoutingIdFieldMapper.class) .item(VersionFieldMapper.class) ); List matching = new ArrayList<>(documentMapper.mappers().getMatchingFieldNames("*")); @@ -340,7 +339,6 @@ public void testEmptyDocumentMapper() { .item(IndexFieldMapper.CONTENT_TYPE) .item(NestedPathFieldMapper.NAME) .item(RoutingFieldMapper.CONTENT_TYPE) - .item(TimeSeriesRoutingIdFieldMapper.NAME) .item(SeqNoFieldMapper.CONTENT_TYPE) .item(SourceFieldMapper.CONTENT_TYPE) .item(VersionFieldMapper.CONTENT_TYPE) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index 0f2380f6c72fb..efab3d67dd17a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -121,8 +121,9 @@ private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesR private static Set builtInMetadataFields() { Set builtInMetadataFields = new HashSet<>(IndicesModule.getBuiltInMetadataFields()); - // Index is not a time-series index, and it will not contain a _tsid field + // Index is not a time-series index, and it will not contain _tsid and _routing_id fields. builtInMetadataFields.remove(TimeSeriesIdFieldMapper.NAME); + builtInMetadataFields.remove(TimeSeriesRoutingIdFieldMapper.NAME); return builtInMetadataFields; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapperTests.java new file mode 100644 index 0000000000000..3c89fdee76a0d --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapperTests.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; + +public class TimeSeriesRoutingIdFieldMapperTests extends MetadataMapperTestCase { + + @Override + protected String fieldName() { + return TimeSeriesRoutingIdFieldMapper.NAME; + } + + @Override + protected boolean isConfigurable() { + return false; + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + // There aren't any parameters + } + + private DocumentMapper createMapper(XContentBuilder mappings) throws IOException { + return createMapperService( + getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.TIME_SERIES.name()) + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "routing path is required") + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), "2021-04-28T00:00:00Z") + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), "2021-04-29T00:00:00Z") + .build(), + mappings + ).documentMapper(); + } + + private static ParsedDocument parseDocument(int hash, DocumentMapper docMapper, CheckedConsumer f) + throws IOException { + // Add the @timestamp field required by DataStreamTimestampFieldMapper for all time series indices + return docMapper.parse(source(TimeSeriesRoutingIdFieldMapper.encode(hash), b -> { + f.accept(b); + b.field("@timestamp", "2021-10-01"); + }, null)); + } + + private static int getRoutingId(ParsedDocument document) { + BytesRef value = document.rootDoc().getBinaryValue(TimeSeriesRoutingIdFieldMapper.NAME); + return TimeSeriesRoutingIdFieldMapper.decode(Uid.decodeId(value.bytes)); + } + + @SuppressWarnings("unchecked") + public void testEnabledInTimeSeriesMode() throws Exception { + DocumentMapper docMapper = createMapper(mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + + int hash = randomInt(); + ParsedDocument doc = parseDocument(hash, docMapper, b -> b.field("a", "value")); + assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); + assertEquals(hash, getRoutingId(doc)); + } + + public void testDisabledInStandardMode() throws Exception { + DocumentMapper docMapper = createMapperService( + getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), + mapping(b -> {}) + ).documentMapper(); + assertThat(docMapper.metadataMapper(TimeSeriesRoutingIdFieldMapper.class), is(nullValue())); + + ParsedDocument doc = docMapper.parse(source("id", b -> b.field("field", "value"), null)); + assertThat(doc.rootDoc().getBinaryValue("_routing_id"), is(nullValue())); + assertThat(doc.rootDoc().get("field"), equalTo("value")); + } + + public void testIncludeInDocumentNotAllowed() throws Exception { + DocumentMapper docMapper = createMapper(mapping(b -> { + b.startObject("a").field("type", "keyword").field("time_series_dimension", true).endObject(); + })); + Exception e = expectThrows( + DocumentParsingException.class, + () -> parseDocument(randomInt(), docMapper, b -> b.field("_routing_id", "foo")) + ); + + assertThat( + e.getCause().getMessage(), + containsString("Field [_routing_id] is a metadata field and cannot be added inside a document") + ); + } +} From cd4fb928ffbcb1f8777cfb6fef67b3b265d87ca1 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Tue, 5 Mar 2024 17:28:54 +0200 Subject: [PATCH 35/51] refactor passthrough test --- .../TSDBPassthroughIndexingIT.java | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java index c7b01abd02b11..aa3fa2a730be3 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java @@ -115,6 +115,15 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase { } """; + private static String getRandomDoc(Instant time) { + return DOC.replace("$time", formatInstant(time)) + .replace("$uid", randomUUID()) + .replace("$name", randomAlphaOfLength(4)) + .replace("$number1", Long.toString(randomLong())) + .replace("$number2", Double.toString(randomDouble())) + .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))); + } + @Override protected Collection> getPlugins() { return List.of(DataStreamsPlugin.class, InternalSettingsPlugin.class); @@ -150,15 +159,7 @@ public void testIndexingGettingAndSearching() throws Exception { Instant time = Instant.now(); for (int i = 0; i < indexingIters; i++) { var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE); - indexRequest.source( - DOC.replace("$time", formatInstant(time)) - .replace("$uid", randomUUID()) - .replace("$name", randomAlphaOfLength(4)) - .replace("$number1", Long.toString(randomLong())) - .replace("$number2", Double.toString(randomDouble())) - .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))), - XContentType.JSON - ); + indexRequest.source(getRandomDoc(time), XContentType.JSON); var indexResponse = client().index(indexRequest).actionGet(); index = indexResponse.getIndex(); String id = indexResponse.getId(); @@ -245,15 +246,7 @@ public void testIndexingGettingAndSearchingShrunkIndex() throws Exception { var bulkRequest = new BulkRequest(dataStreamName); for (int i = 0; i < numBulkItems; i++) { var indexRequest = new IndexRequest(dataStreamName).opType(DocWriteRequest.OpType.CREATE); - indexRequest.source( - DOC.replace("$time", formatInstant(time)) - .replace("$uid", randomUUID()) - .replace("$name", randomAlphaOfLength(4)) - .replace("$number1", Long.toString(randomLong())) - .replace("$number2", Double.toString(randomDouble())) - .replace("$ip", InetAddresses.toAddrString(randomIp(randomBoolean()))), - XContentType.JSON - ); + indexRequest.source(getRandomDoc(time), XContentType.JSON); bulkRequest.add(indexRequest); time = time.plusMillis(1); } From ed7a007c6cd1a48cbd0d286dcf6338d085cf7172 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 7 Mar 2024 10:54:51 +0200 Subject: [PATCH 36/51] address comments --- .../action/PainlessExecuteAction.java | 10 +--- .../elasticsearch/action/DocWriteRequest.java | 7 +-- .../action/bulk/BulkOperation.java | 3 +- .../action/delete/DeleteRequest.java | 7 +-- .../action/index/IndexRequest.java | 12 ++--- .../action/update/UpdateRequest.java | 7 +-- .../cluster/routing/IndexRouting.java | 52 ++++++++++++------- .../org/elasticsearch/index/IndexMode.java | 10 ++-- .../elasticsearch/index/IndexVersions.java | 2 +- .../index/codec/PerFieldMapperCodec.java | 2 +- .../elasticsearch/index/mapper/IdLoader.java | 10 ++-- .../index/mapper/TimeSeriesIdFieldMapper.java | 4 +- ... => TimeSeriesRoutingHashFieldMapper.java} | 39 ++++++-------- .../mapper/TsidExtractingIdFieldMapper.java | 12 ++--- .../elasticsearch/indices/IndicesModule.java | 4 +- .../search/DefaultSearchContext.java | 2 +- .../cluster/routing/IndexRoutingTests.java | 20 +++---- .../mapper/FieldFilterMapperPluginTests.java | 4 +- .../index/mapper/IdLoaderTests.java | 7 ++- .../mapper/TimeSeriesIdFieldMapperTests.java | 16 +++--- ...imeSeriesRoutingHashFieldMapperTests.java} | 24 ++++----- .../TsidExtractingIdFieldMapperTests.java | 28 +++++++--- .../indices/IndicesModuleTests.java | 4 +- .../recovery/RecoverySourceHandlerTests.java | 21 +++++--- .../ESIndexLevelReplicationTestCase.java | 2 +- .../test/InternalTestCluster.java | 2 +- 26 files changed, 159 insertions(+), 152 deletions(-) rename server/src/main/java/org/elasticsearch/index/mapper/{TimeSeriesRoutingIdFieldMapper.java => TimeSeriesRoutingHashFieldMapper.java} (71%) rename server/src/test/java/org/elasticsearch/index/mapper/{TimeSeriesRoutingIdFieldMapperTests.java => TimeSeriesRoutingHashFieldMapperTests.java} (81%) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 8c49fa1240ca1..874b2316406ef 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -56,13 +56,11 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.OnScriptError; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; @@ -812,13 +810,7 @@ private static Response prepareRamIndex( XContentType xContentType = request.contextSetup.xContentType; String id; if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - if (indexService.getIndexSettings() - .getIndexVersionCreated() - .onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { - id = TimeSeriesRoutingIdFieldMapper.encode(0); - } else { - id = null; - } + id = null; // The id gets auto generated for time series indices. } else { id = "_id"; } diff --git a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java index c4ac655caac6d..7f3578ce9f16f 100644 --- a/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/DocWriteRequest.java @@ -161,12 +161,7 @@ public interface DocWriteRequest extends IndicesRequest, Accountable { /** * Finalize the request before executing or routing it. */ - void preProcess(IndexRouting indexRouting); - - /** - * Finalize the request after executing or routing it. - */ - void postProcess(IndexRouting indexRouting); + void process(IndexRouting indexRouting); /** * Pick the appropriate shard id to receive this request. diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index 70e7e642cf889..1d95f430d5c7e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -162,9 +162,8 @@ private Map> groupRequestsByShards(ClusterState c continue; } IndexRouting indexRouting = concreteIndices.routing(concreteIndex); - docWriteRequest.preProcess(indexRouting); + docWriteRequest.process(indexRouting); int shardId = docWriteRequest.route(indexRouting); - docWriteRequest.postProcess(indexRouting); List shardRequests = requestsByShard.computeIfAbsent( new ShardId(concreteIndex, shardId), shard -> new ArrayList<>() diff --git a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java index 6eb7bcac64861..e4b8edea58114 100644 --- a/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java +++ b/server/src/main/java/org/elasticsearch/action/delete/DeleteRequest.java @@ -237,12 +237,7 @@ public boolean isRequireDataStream() { } @Override - public void preProcess(IndexRouting indexRouting) { - // Nothing to do - } - - @Override - public void postProcess(IndexRouting indexRouting) { + public void process(IndexRouting indexRouting) { // Nothing to do } diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index c2ae4e80f6136..8f60146a5491e 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -684,13 +684,8 @@ public VersionType versionType() { } @Override - public void preProcess(IndexRouting indexRouting) { - indexRouting.preProcess(this); - } - - @Override - public void postProcess(IndexRouting indexRouting) { - indexRouting.postProcess(this); + public void process(IndexRouting indexRouting) { + indexRouting.process(this); } /** @@ -886,7 +881,8 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, Metadata metadata) { @Override public int route(IndexRouting indexRouting) { - return indexRouting.indexShard(id, routing, contentType, source); + assert routing == null : routing; + return indexRouting.indexShard(id, routing, contentType, source, this::routing); } public IndexRequest setRequireAlias(boolean requireAlias) { diff --git a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java index 9f7b7bf0720b5..36b6cc6aa9964 100644 --- a/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java +++ b/server/src/main/java/org/elasticsearch/action/update/UpdateRequest.java @@ -848,12 +848,7 @@ public boolean isRequireDataStream() { } @Override - public void preProcess(IndexRouting indexRouting) { - // Nothing to do - } - - @Override - public void postProcess(IndexRouting indexRouting) { + public void process(IndexRouting indexRouting) { // Nothing to do } diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index 234d34a1e8514..bb839bd72c822 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -22,7 +22,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.transport.Transports; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParser.Token; @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import java.util.function.IntConsumer; import java.util.function.IntSupplier; import java.util.function.Predicate; @@ -70,15 +71,19 @@ private IndexRouting(IndexMetadata metadata) { this.routingFactor = metadata.getRoutingFactor(); } - public void preProcess(IndexRequest indexRequest) {} - - public void postProcess(IndexRequest indexRequest) {} + public void process(IndexRequest indexRequest) {} /** * Called when indexing a document to generate the shard id that should contain * a document with the provided parameters. */ - public abstract int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source); + public abstract int indexShard( + String id, + @Nullable String routing, + XContentType sourceType, + BytesReference source, + Consumer routingHashRecorder + ); /** * Called when updating a document to generate the shard id that should contain @@ -145,7 +150,7 @@ private abstract static class IdAndRoutingOnly extends IndexRouting { protected abstract int shardId(String id, @Nullable String routing); @Override - public void preProcess(IndexRequest indexRequest) { + public void process(IndexRequest indexRequest) { if ("".equals(indexRequest.id())) { throw new IllegalArgumentException("if _id is specified it must not be empty"); } @@ -157,7 +162,13 @@ public void preProcess(IndexRequest indexRequest) { } @Override - public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + public int indexShard( + String id, + @Nullable String routing, + XContentType sourceType, + BytesReference source, + Consumer routingHashRecorder + ) { if (id == null) { throw new IllegalStateException("id is required and should have been set by process"); } @@ -243,14 +254,14 @@ public static class ExtractFromSource extends IndexRouting { private final XContentParserConfiguration parserConfig; private final boolean indexTracksRoutingId; - private int routingId = 0; + private int routingHash = 0; ExtractFromSource(IndexMetadata metadata) { super(metadata); if (metadata.isRoutingPartitionedIndex()) { throw new IllegalArgumentException("routing_partition_size is incompatible with routing_path"); } - indexTracksRoutingId = metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID); + indexTracksRoutingId = metadata.getCreationVersion().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID); List routingPaths = metadata.getRoutingPaths(); isRoutingPath = Regex.simpleMatcher(routingPaths.toArray(String[]::new)); this.parserConfig = XContentParserConfiguration.EMPTY.withFiltering(Set.copyOf(routingPaths), null, true); @@ -261,17 +272,20 @@ public boolean matchesField(String fieldName) { } @Override - public void postProcess(IndexRequest indexRequest) { - if (indexRequest.id() == null && indexTracksRoutingId) { - indexRequest.id(TimeSeriesRoutingIdFieldMapper.encode(routingId)); - } - } - - @Override - public int indexShard(String id, @Nullable String routing, XContentType sourceType, BytesReference source) { + public int indexShard( + String id, + @Nullable String routing, + XContentType sourceType, + BytesReference source, + Consumer routingHashRecorder + ) { assert Transports.assertNotTransportThread("parsing the _source can get slow"); checkNoRouting(routing); - return hashToShardId(hashSource(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty)); + int hash = hashSource(sourceType, source).buildHash(IndexRouting.ExtractFromSource::defaultOnEmpty); + if (indexTracksRoutingId) { + routingHashRecorder.accept(TimeSeriesRoutingHashFieldMapper.encode(hash)); + } + return hashToShardId(hash); } public String createId(XContentType sourceType, BytesReference source, byte[] suffix) { @@ -379,7 +393,7 @@ private int buildHash(IntSupplier onEmpty) { hash = 31 * hash + thisHash; prev = next; } - routingId = hash; + routingHash = hash; return hash; } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 969b5209daa9e..a5bad7b229fb2 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -28,7 +28,7 @@ import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.TsidExtractingIdFieldMapper; import java.io.IOException; @@ -94,7 +94,7 @@ public MetadataFieldMapper timeSeriesIdFieldMapper() { } @Override - public MetadataFieldMapper timeSeriesRoutingIdFieldMapper() { + public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { // non time-series indices must not have a TimeSeriesRoutingIdFieldMapper return null; } @@ -193,8 +193,8 @@ public MetadataFieldMapper timeSeriesIdFieldMapper() { } @Override - public MetadataFieldMapper timeSeriesRoutingIdFieldMapper() { - return TimeSeriesRoutingIdFieldMapper.INSTANCE; + public MetadataFieldMapper timeSeriesRoutingHashFieldMapper() { + return TimeSeriesRoutingHashFieldMapper.INSTANCE; } public IdFieldMapper idFieldMapperWithoutFieldData() { @@ -339,7 +339,7 @@ public String getName() { * the _tsid field. The field mapper will be added to the list of the metadata * field mappers for the index. */ - public abstract MetadataFieldMapper timeSeriesRoutingIdFieldMapper(); + public abstract MetadataFieldMapper timeSeriesRoutingHashFieldMapper(); /** * How {@code time_series_dimension} fields are handled by indices in this mode. diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 8428d20a2b96e..bca7b963becaa 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -102,7 +102,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion UPGRADE_LUCENE_9_9_2 = def(8_502_00_0, Version.LUCENE_9_9_2); public static final IndexVersion TIME_SERIES_ID_HASHING = def(8_502_00_1, Version.LUCENE_9_9_2); public static final IndexVersion UPGRADE_TO_LUCENE_9_10 = def(8_503_00_0, Version.LUCENE_9_10_0); - public static final IndexVersion TIME_SERIES_ROUTING_ID_IN_ID = def(8_504_00_0, Version.LUCENE_9_10_0); + public static final IndexVersion TIME_SERIES_ROUTING_HASH_IN_ID = def(8_504_00_0, Version.LUCENE_9_10_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java index ba7d4c034044f..397b7180e791b 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java @@ -116,7 +116,7 @@ boolean useTSDBDocValuesFormat(final String field) { private boolean excludeFields(String fieldName) { // Avoid using tsdb codec for fields like _seq_no, _primary_term. // But _tsid and _routing_path should always use the tsdb codec. - return fieldName.startsWith("_") && fieldName.equals("_tsid") == false && fieldName.equals("_routing_id") == false; + return fieldName.startsWith("_") && fieldName.equals("_tsid") == false && fieldName.equals("_ts_routing_hash") == false; } private boolean isTimeSeriesModeIndex() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java index 5d76fc86733f8..ef15af93f6e34 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IdLoader.java @@ -107,12 +107,12 @@ public IdLoader.Leaf leaf(LeafStoredFieldLoader loader, LeafReader reader, int[] var routingBuilder = builders[i]; ids[i] = TsidExtractingIdFieldMapper.createId(false, routingBuilder, tsid, timestamp, new byte[16]); } else { - SortedDocValues routingIdDocValues = DocValues.getSorted(reader, TimeSeriesRoutingIdFieldMapper.NAME); - found = routingIdDocValues.advanceExact(docId); + SortedDocValues routingHashDocValues = DocValues.getSorted(reader, TimeSeriesRoutingHashFieldMapper.NAME); + found = routingHashDocValues.advanceExact(docId); assert found; - BytesRef routingIdBytes = routingIdDocValues.lookupOrd(routingIdDocValues.ordValue()); - int routingId = TimeSeriesRoutingIdFieldMapper.decode(Uid.decodeId(routingIdBytes.bytes)); - ids[i] = TsidExtractingIdFieldMapper.createId(routingId, tsid, timestamp); + BytesRef routingHashBytes = routingHashDocValues.lookupOrd(routingHashDocValues.ordValue()); + int routingHash = TimeSeriesRoutingHashFieldMapper.decode(Uid.decodeId(routingHashBytes.bytes)); + ids[i] = TsidExtractingIdFieldMapper.createId(routingHash, tsid, timestamp); } } return new TsIdLeaf(docIdsInLeaf, ids); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 5ddf779efe085..112b3ec96b39e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -142,7 +142,9 @@ public void postParse(DocumentParserContext context) throws IOException { context.doc().add(new SortedDocValuesField(fieldType().name(), timeSeriesId)); TsidExtractingIdFieldMapper.createField( context, - getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID) ? timeSeriesIdBuilder.routingBuilder : null, + getIndexVersionCreated(context).before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID) + ? timeSeriesIdBuilder.routingBuilder + : null, timeSeriesId ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java similarity index 71% rename from server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java rename to server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java index a223d9ac3cfa6..2eff954844ebc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapper.java @@ -26,32 +26,25 @@ import java.util.Collections; /** - * Mapper for the {@code _seq_no} field. + * Mapper for the {@code _ts_routing_hash} field. * - * We expect to use the seq# for sorting, during collision checking and for - * doing range searches. Therefore the {@code _seq_no} field is stored both - * as a numeric doc value and as numeric indexed field. - * - * This mapper also manages the primary term field, which has no ES named - * equivalent. The primary term is only used during collision after receiving - * identical seq# values for two document copies. The primary term is stored as - * a doc value field without being indexed, since it is only intended for use - * as a key-value lookup. - + * The field contains the routing hash, as calculated in coordinating nodes for docs in time-series indexes. + * It's stored to be retrieved and added as a prefix when reconstructing the _id field in search queries. + * The prefix can then used for routing Get and Delete requests (by doc id) to the right shard. */ -public class TimeSeriesRoutingIdFieldMapper extends MetadataFieldMapper { +public class TimeSeriesRoutingHashFieldMapper extends MetadataFieldMapper { - public static final String NAME = "_routing_id"; + public static final String NAME = "_ts_routing_hash"; - public static final TimeSeriesRoutingIdFieldMapper INSTANCE = new TimeSeriesRoutingIdFieldMapper(); + public static final TimeSeriesRoutingHashFieldMapper INSTANCE = new TimeSeriesRoutingHashFieldMapper(); - public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesRoutingIdFieldMapper()); + public static final TypeParser PARSER = new FixedTypeParser(c -> c.getIndexSettings().getMode().timeSeriesRoutingHashFieldMapper()); - static final class TimeSeriesRoutingIdFieldType extends MappedFieldType { + static final class TimeSeriesRoutingHashFieldType extends MappedFieldType { - private static final TimeSeriesRoutingIdFieldType INSTANCE = new TimeSeriesRoutingIdFieldType(); + private static final TimeSeriesRoutingHashFieldType INSTANCE = new TimeSeriesRoutingHashFieldType(); - private TimeSeriesRoutingIdFieldType() { + private TimeSeriesRoutingHashFieldType() { super(NAME, false, false, true, TextSearchInfo.NONE, Collections.emptyMap()); } @@ -84,16 +77,16 @@ public Query termQuery(Object value, SearchExecutionContext context) { } } - private TimeSeriesRoutingIdFieldMapper() { - super(TimeSeriesRoutingIdFieldType.INSTANCE); + private TimeSeriesRoutingHashFieldMapper() { + super(TimeSeriesRoutingHashFieldType.INSTANCE); } @Override public void postParse(DocumentParserContext context) { if (context.indexSettings().getMode() == IndexMode.TIME_SERIES - && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID) - && context.sourceToParse().id() != null) { - var field = new SortedDocValuesField(NAME, Uid.encodeId(context.sourceToParse().id())); + && context.indexSettings().getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { + String routingHash = context.sourceToParse().routing(); + var field = new SortedDocValuesField(NAME, Uid.encodeId(routingHash != null ? routingHash : encode(0))); context.rootDoc().add(field); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 5d874d75ad9f0..ea5caae8cb7a9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -82,12 +82,12 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra ) ); } - } else if (context.sourceToParse().id() != null) { - int routingId = TimeSeriesRoutingIdFieldMapper.decode(context.sourceToParse().id()); - id = createId(routingId, tsid, timestamp); + } else if (context.sourceToParse().routing() != null) { + int routingHash = TimeSeriesRoutingHashFieldMapper.decode(context.sourceToParse().routing()); + id = createId(routingHash, tsid, timestamp); } else { throw new IllegalArgumentException( - "_id was null but must be set because index [" + "_ts_routing_hash was null but must be set because index [" + context.indexSettings().getIndexMetadata().getIndex().getName() + "] is in time_series mode" ); @@ -98,12 +98,12 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra context.doc().add(new StringField(NAME, uidEncoded, Field.Store.YES)); } - public static String createId(int routingId, BytesRef tsid, long timestamp) { + public static String createId(int routingHash, BytesRef tsid, long timestamp) { Hash128 hash = new Hash128(); MurmurHash3.hash128(tsid.bytes, tsid.offset, tsid.length, SEED, hash); byte[] bytes = new byte[20]; - ByteUtils.writeIntLE(routingId, bytes, 0); + ByteUtils.writeIntLE(routingHash, bytes, 0); ByteUtils.writeLongLE(hash.h1, bytes, 4); ByteUtils.writeLongBE(timestamp, bytes, 12); // Big Ending shrinks the inverted index by ~37% diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 041d68261d9cc..b94c95834f65a 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -60,7 +60,7 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; @@ -248,7 +248,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(IdFieldMapper.NAME, IdFieldMapper.PARSER); builtInMetadataMappers.put(RoutingFieldMapper.NAME, RoutingFieldMapper.PARSER); builtInMetadataMappers.put(TimeSeriesIdFieldMapper.NAME, TimeSeriesIdFieldMapper.PARSER); - builtInMetadataMappers.put(TimeSeriesRoutingIdFieldMapper.NAME, TimeSeriesRoutingIdFieldMapper.PARSER); + builtInMetadataMappers.put(TimeSeriesRoutingHashFieldMapper.NAME, TimeSeriesRoutingHashFieldMapper.PARSER); builtInMetadataMappers.put(IndexFieldMapper.NAME, IndexFieldMapper.PARSER); builtInMetadataMappers.put(SourceFieldMapper.NAME, SourceFieldMapper.PARSER); builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 0bf4fd1b951a2..0e6800b9c8d48 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -900,7 +900,7 @@ public IdLoader newIdLoader() { if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { IndexRouting.ExtractFromSource indexRouting = null; List routingPaths = null; - if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID)) { + if (indexService.getIndexSettings().getIndexVersionCreated().before(IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID)) { indexRouting = (IndexRouting.ExtractFromSource) indexService.getIndexSettings().getIndexRouting(); routingPaths = indexService.getMetadata().getRoutingPaths(); for (String routingField : routingPaths) { diff --git a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java index 3360ab86ff01b..d76e874c7061b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/routing/IndexRoutingTests.java @@ -47,7 +47,7 @@ public void testSimpleRoutingRejectsEmptyId() { IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() ); IndexRequest req = new IndexRequest().id(""); - Exception e = expectThrows(IllegalArgumentException.class, () -> indexRouting.preProcess(req)); + Exception e = expectThrows(IllegalArgumentException.class, () -> indexRouting.process(req)); assertThat(e.getMessage(), equalTo("if _id is specified it must not be empty")); } @@ -57,7 +57,7 @@ public void testSimpleRoutingAcceptsId() { ); String id = randomAlphaOfLength(10); IndexRequest req = new IndexRequest().id(id); - indexRouting.preProcess(req); + indexRouting.process(req); assertThat(req.id(), equalTo(id)); assertThat(req.getAutoGeneratedTimestamp(), equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP)); } @@ -67,7 +67,7 @@ public void testSimpleRoutingAssignedRandomId() { IndexMetadata.builder("test").settings(settings(IndexVersion.current())).numberOfShards(2).numberOfReplicas(1).build() ); IndexRequest req = new IndexRequest(); - indexRouting.preProcess(req); + indexRouting.process(req); assertThat(req.id(), not(nullValue())); assertThat(req.getAutoGeneratedTimestamp(), not(equalTo(IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP))); } @@ -457,7 +457,7 @@ public void testRequiredRouting() { */ private int shardIdFromSimple(IndexRouting indexRouting, String id, @Nullable String routing) { return switch (between(0, 3)) { - case 0 -> indexRouting.indexShard(id, routing, null, null); + case 0 -> indexRouting.indexShard(id, routing, null, null, null); case 1 -> indexRouting.updateShard(id, routing); case 2 -> indexRouting.deleteShard(id, routing); case 3 -> indexRouting.getShard(id, routing); @@ -469,7 +469,7 @@ public void testRoutingAllowsId() { IndexRouting indexRouting = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); String id = randomAlphaOfLength(5); IndexRequest req = new IndexRequest().id(id); - indexRouting.preProcess(req); + indexRouting.process(req); assertThat(req.id(), equalTo(id)); } @@ -482,7 +482,7 @@ public void testRoutingAllowsId() { public void testRoutingPathLeavesIdNull() { IndexRouting indexRouting = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); IndexRequest req = new IndexRequest(); - indexRouting.preProcess(req); + indexRouting.process(req); assertThat(req.id(), nullValue()); } @@ -490,7 +490,7 @@ public void testRoutingPathEmptySource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), randomAlphaOfLength(5)); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of())) + () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of()), null) ); assertThat(e.getMessage(), equalTo("Error extracting routing: source didn't contain any routing fields")); } @@ -499,7 +499,7 @@ public void testRoutingPathMismatchSource() throws IOException { IndexRouting routing = indexRoutingForPath(between(1, 5), "foo"); Exception e = expectThrows( IllegalArgumentException.class, - () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of("bar", "dog"))) + () -> routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, source(Map.of("bar", "dog")), null) ); assertThat(e.getMessage(), equalTo("Error extracting routing: source didn't contain any routing fields")); } @@ -520,7 +520,7 @@ public void testRoutingIndexWithRouting() throws IOException { String docRouting = randomAlphaOfLength(5); Exception e = expectThrows( IllegalArgumentException.class, - () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, XContentType.JSON, source) + () -> indexRouting.indexShard(randomAlphaOfLength(5), docRouting, XContentType.JSON, source, null) ); assertThat( e.getMessage(), @@ -649,7 +649,7 @@ private IndexRouting indexRoutingForPath(IndexVersion createdVersion, int shards private void assertIndexShard(IndexRouting routing, Map source, int expectedShard) throws IOException { byte[] suffix = randomSuffix(); BytesReference sourceBytes = source(source); - assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, sourceBytes), equalTo(expectedShard)); + assertThat(routing.indexShard(randomAlphaOfLength(5), null, XContentType.JSON, sourceBytes, s -> {}), equalTo(expectedShard)); IndexRouting.ExtractFromSource r = (IndexRouting.ExtractFromSource) routing; String idFromSource = r.createId(XContentType.JSON, sourceBytes, suffix); assertThat(shardIdForReadFromSourceExtracting(routing, idFromSource), equalTo(expectedShard)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java index efab3d67dd17a..2b8be2882c409 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldFilterMapperPluginTests.java @@ -121,9 +121,9 @@ private static void assertFieldCaps(FieldCapabilitiesResponse fieldCapabilitiesR private static Set builtInMetadataFields() { Set builtInMetadataFields = new HashSet<>(IndicesModule.getBuiltInMetadataFields()); - // Index is not a time-series index, and it will not contain _tsid and _routing_id fields. + // Index is not a time-series index, and it will not contain _tsid and _ts_routing_hash fields. builtInMetadataFields.remove(TimeSeriesIdFieldMapper.NAME); - builtInMetadataFields.remove(TimeSeriesRoutingIdFieldMapper.NAME); + builtInMetadataFields.remove(TimeSeriesRoutingHashFieldMapper.NAME); return builtInMetadataFields; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java index fb5004307d48f..e4ce40d4c7c29 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IdLoaderTests.java @@ -209,7 +209,7 @@ private void prepareIndexReader( } Sort sort = new Sort( new SortField(TimeSeriesIdFieldMapper.NAME, SortField.Type.STRING, false), - new SortField(TimeSeriesRoutingIdFieldMapper.NAME, SortField.Type.STRING, false), + new SortField(TimeSeriesRoutingHashFieldMapper.NAME, SortField.Type.STRING, false), new SortedNumericSortField(DataStreamTimestampFieldMapper.DEFAULT_PATH, SortField.Type.LONG, true) ); config.setIndexSort(sort); @@ -241,7 +241,10 @@ private static void indexDoc(IndexWriter iw, Doc doc, int routingHash) throws IO BytesRef tsid = builder.buildTsidHash().toBytesRef(); fields.add(new SortedDocValuesField(TimeSeriesIdFieldMapper.NAME, tsid)); fields.add( - new SortedDocValuesField(TimeSeriesRoutingIdFieldMapper.NAME, Uid.encodeId(TimeSeriesRoutingIdFieldMapper.encode(routingHash))) + new SortedDocValuesField( + TimeSeriesRoutingHashFieldMapper.NAME, + Uid.encodeId(TimeSeriesRoutingHashFieldMapper.encode(routingHash)) + ) ); iw.addDocument(fields); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 4f79847e9b15a..7c689e88edf13 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -22,11 +22,13 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.hamcrest.Matchers.containsString; @@ -667,12 +669,12 @@ public void testParseWithDynamicMapping() { .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") .build(); MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); - SourceToParse source = new SourceToParse(TimeSeriesRoutingIdFieldMapper.encode(0), new BytesArray(""" + SourceToParse source = new SourceToParse(null, new BytesArray(""" { "@timestamp": 1609459200000, "dim": "6a841a21", "value": 100 - }"""), XContentType.JSON); + }"""), XContentType.JSON, TimeSeriesRoutingHashFieldMapper.encode(0), Map.of(), DocumentSizeObserver.EMPTY_INSTANCE); Engine.Index index = IndexShard.prepareIndex( mapper, source, @@ -690,18 +692,18 @@ public void testParseWithDynamicMapping() { assertNotNull(index.parsedDoc().dynamicMappingsUpdate()); } - public void testParseWithDynamicMappingInvalidId() { + public void testParseWithDynamicMappingInvalidRoutingHash() { Settings indexSettings = Settings.builder() .put(IndexSettings.MODE.getKey(), "time_series") .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "dim") .build(); MapperService mapper = createMapperService(IndexVersion.current(), indexSettings, () -> false); - SourceToParse source = new SourceToParse("no-such-id-exists", new BytesArray(""" + SourceToParse source = new SourceToParse(null, new BytesArray(""" { "@timestamp": 1609459200000, "dim": "6a841a21", "value": 100 - }"""), XContentType.JSON); + }"""), XContentType.JSON, "no such routing hash", Map.of(), DocumentSizeObserver.EMPTY_INSTANCE); var failure = expectThrows(DocumentParsingException.class, () -> { IndexShard.prepareIndex( mapper, @@ -718,7 +720,7 @@ public void testParseWithDynamicMappingInvalidId() { System.nanoTime() ); }); - assertThat(failure.getMessage(), equalTo("[5:1] failed to parse: Last unit does not have enough valid bits")); + assertThat(failure.getMessage(), equalTo("[5:1] failed to parse: Illegal base64 character 20")); } public void testParseWithDynamicMappingNullId() { @@ -751,7 +753,7 @@ public void testParseWithDynamicMappingNullId() { }); assertThat( failure.getMessage(), - equalTo("[5:1] failed to parse: _id was null but must be set because index [index] is in time_series mode") + equalTo("[5:1] failed to parse: _ts_routing_hash was null but must be set because index [index] is in time_series mode") ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java similarity index 81% rename from server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapperTests.java rename to server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java index 3c89fdee76a0d..df5ff9a8fe7e5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesRoutingHashFieldMapperTests.java @@ -22,11 +22,11 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.nullValue; -public class TimeSeriesRoutingIdFieldMapperTests extends MetadataMapperTestCase { +public class TimeSeriesRoutingHashFieldMapperTests extends MetadataMapperTestCase { @Override protected String fieldName() { - return TimeSeriesRoutingIdFieldMapper.NAME; + return TimeSeriesRoutingHashFieldMapper.NAME; } @Override @@ -53,15 +53,15 @@ private DocumentMapper createMapper(XContentBuilder mappings) throws IOException private static ParsedDocument parseDocument(int hash, DocumentMapper docMapper, CheckedConsumer f) throws IOException { // Add the @timestamp field required by DataStreamTimestampFieldMapper for all time series indices - return docMapper.parse(source(TimeSeriesRoutingIdFieldMapper.encode(hash), b -> { + return docMapper.parse(source(null, b -> { f.accept(b); b.field("@timestamp", "2021-10-01"); - }, null)); + }, TimeSeriesRoutingHashFieldMapper.encode(hash))); } - private static int getRoutingId(ParsedDocument document) { - BytesRef value = document.rootDoc().getBinaryValue(TimeSeriesRoutingIdFieldMapper.NAME); - return TimeSeriesRoutingIdFieldMapper.decode(Uid.decodeId(value.bytes)); + private static int getRoutingHash(ParsedDocument document) { + BytesRef value = document.rootDoc().getBinaryValue(TimeSeriesRoutingHashFieldMapper.NAME); + return TimeSeriesRoutingHashFieldMapper.decode(Uid.decodeId(value.bytes)); } @SuppressWarnings("unchecked") @@ -73,7 +73,7 @@ public void testEnabledInTimeSeriesMode() throws Exception { int hash = randomInt(); ParsedDocument doc = parseDocument(hash, docMapper, b -> b.field("a", "value")); assertThat(doc.rootDoc().getField("a").binaryValue(), equalTo(new BytesRef("value"))); - assertEquals(hash, getRoutingId(doc)); + assertEquals(hash, getRoutingHash(doc)); } public void testDisabledInStandardMode() throws Exception { @@ -81,10 +81,10 @@ public void testDisabledInStandardMode() throws Exception { getIndexSettingsBuilder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()).build(), mapping(b -> {}) ).documentMapper(); - assertThat(docMapper.metadataMapper(TimeSeriesRoutingIdFieldMapper.class), is(nullValue())); + assertThat(docMapper.metadataMapper(TimeSeriesRoutingHashFieldMapper.class), is(nullValue())); ParsedDocument doc = docMapper.parse(source("id", b -> b.field("field", "value"), null)); - assertThat(doc.rootDoc().getBinaryValue("_routing_id"), is(nullValue())); + assertThat(doc.rootDoc().getBinaryValue("_ts_routing_hash"), is(nullValue())); assertThat(doc.rootDoc().get("field"), equalTo("value")); } @@ -94,12 +94,12 @@ public void testIncludeInDocumentNotAllowed() throws Exception { })); Exception e = expectThrows( DocumentParsingException.class, - () -> parseDocument(randomInt(), docMapper, b -> b.field("_routing_id", "foo")) + () -> parseDocument(randomInt(), docMapper, b -> b.field("_ts_routing_hash", "foo")) ); assertThat( e.getCause().getMessage(), - containsString("Field [_routing_id] is a metadata field and cannot be added inside a document") + containsString("Field [_ts_routing_hash] is a metadata field and cannot be added inside a document") ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 524d862ca5eee..f40a3a8d621e5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -27,6 +28,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.stream.Stream; import static org.hamcrest.Matchers.equalTo; @@ -692,7 +694,7 @@ public TsidExtractingIdFieldMapperTests(@Named("testCase") TestCase testCase) { public void testExpectedId() throws IOException { assertThat( - parse(TimeSeriesRoutingIdFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.source).id(), + parse(TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.source).id(), equalTo(testCase.expectedId) ); } @@ -705,25 +707,35 @@ public void testEquivalentSources() throws IOException { MapperService mapperService = mapperService(); for (CheckedConsumer equivalent : testCase.equivalentSources) { assertThat( - parse(TimeSeriesRoutingIdFieldMapper.encode(ROUTING_HASH), mapperService, equivalent).id(), + parse(TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), mapperService, equivalent).id(), equalTo(testCase.expectedId) ); } } - private ParsedDocument parse(@Nullable String id, MapperService mapperService, CheckedConsumer source) - throws IOException { + private ParsedDocument parse( + @Nullable String routingHash, + MapperService mapperService, + CheckedConsumer source + ) throws IOException { try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { builder.startObject(); source.accept(builder); builder.endObject(); - SourceToParse sourceToParse = new SourceToParse(id, BytesReference.bytes(builder), builder.contentType()); + SourceToParse sourceToParse = new SourceToParse( + null, + BytesReference.bytes(builder), + builder.contentType(), + routingHash, + Map.of(), + DocumentSizeObserver.EMPTY_INSTANCE + ); return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); } } public void testRoutingPathCompliant() throws IOException { - IndexVersion version = IndexVersions.TIME_SERIES_ROUTING_ID_IN_ID; + IndexVersion version = IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID; IndexRouting indexRouting = createIndexSettings(version, indexSettings(version)).getIndexRouting(); assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(SHARD_ID)); assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(SHARD_ID)); @@ -787,7 +799,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { public void testSourceDescription() throws IOException { assertThat(TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext()), equalTo("a time series document")); - ParsedDocument d = parse(TimeSeriesRoutingIdFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.randomSource()); + ParsedDocument d = parse(TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.randomSource()); IndexableField timestamp = d.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(timestamp)), @@ -818,7 +830,7 @@ private TestDocumentParserContext documentParserContext(IndexableField... fields public void testParsedDescription() throws IOException { assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription( - parse(TimeSeriesRoutingIdFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.randomSource()) + parse(TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.randomSource()) ), equalTo("[" + testCase.expectedId + "][" + testCase.expectedTsid + "@" + testCase.expectedTimestamp + "]") ); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index ea35d1708abb4..cade1e66c7fc7 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -30,7 +30,7 @@ import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.index.mapper.TimeSeriesIdFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.test.ESTestCase; @@ -81,7 +81,7 @@ public Map getMetadataMappers() { IdFieldMapper.NAME, RoutingFieldMapper.NAME, TimeSeriesIdFieldMapper.NAME, - TimeSeriesRoutingIdFieldMapper.NAME, + TimeSeriesRoutingHashFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, NestedPathFieldMapper.NAME, diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 3a4e1daa1958d..44f57cf0768d4 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -56,7 +56,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.index.mapper.TimeSeriesRoutingIdFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.seqno.ReplicationTracker; import org.elasticsearch.index.seqno.RetentionLease; import org.elasticsearch.index.seqno.RetentionLeases; @@ -73,6 +73,7 @@ import org.elasticsearch.indices.recovery.plan.PeerOnlyRecoveryPlannerService; import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.indices.recovery.plan.ShardRecoveryPlan; +import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.test.CorruptionUtils; import org.elasticsearch.test.DummyShardLock; @@ -518,11 +519,19 @@ private TimeSeriesModeIndexOpFactory() throws IOException { @Override public Engine.Index createIndexOp(int docIdent) { - SourceToParse source = new SourceToParse(TimeSeriesRoutingIdFieldMapper.encode(0), new BytesArray(Strings.format(""" - { - "@timestamp": %s, - "dim": "dim" - }""", docIdent)), XContentType.JSON); + SourceToParse source = new SourceToParse( + null, + new BytesArray(Strings.format(""" + { + "@timestamp": %s, + "dim": "dim" + }""", docIdent)), + XContentType.JSON, + TimeSeriesRoutingHashFieldMapper.encode(0), + Map.of(), + DocumentSizeObserver.EMPTY_INSTANCE + + ); return IndexShard.prepareIndex( mapper, source, diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index 287e3b4350d7c..8ac0e4952c55e 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -872,7 +872,7 @@ private void executeShardBulkOnPrimary( ) { for (BulkItemRequest itemRequest : request.items()) { if (itemRequest.request() instanceof IndexRequest) { - itemRequest.request().preProcess(primary.indexSettings().getIndexRouting()); + itemRequest.request().process(primary.indexSettings().getIndexRouting()); } } final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); diff --git a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java index 16320b3b26301..949679280a53b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/elasticsearch/test/InternalTestCluster.java @@ -2326,7 +2326,7 @@ synchronized String routingKeyForShard(Index index, int shard, Random random) { IndexRouting indexRouting = IndexRouting.fromIndexMetadata(clusterState.metadata().getIndexSafe(index)); while (true) { String routing = RandomStrings.randomAsciiLettersOfLength(random, 10); - if (shard == indexRouting.indexShard("id", routing, null, null)) { + if (shard == indexRouting.indexShard("id", routing, null, null, null)) { return routing; } } From 0823128f922870190b805251f2e89c19c4808877 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 7 Mar 2024 10:57:34 +0200 Subject: [PATCH 37/51] remove redundant variable --- .../java/org/elasticsearch/cluster/routing/IndexRouting.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java index bb839bd72c822..be74c5803eeb4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/IndexRouting.java @@ -254,8 +254,6 @@ public static class ExtractFromSource extends IndexRouting { private final XContentParserConfiguration parserConfig; private final boolean indexTracksRoutingId; - private int routingHash = 0; - ExtractFromSource(IndexMetadata metadata) { super(metadata); if (metadata.isRoutingPartitionedIndex()) { @@ -393,7 +391,6 @@ private int buildHash(IntSupplier onEmpty) { hash = 31 * hash + thisHash; prev = next; } - routingHash = hash; return hash; } } From 1ce66186f38649070595cab1e8429887491f7fe7 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 7 Mar 2024 11:29:44 +0200 Subject: [PATCH 38/51] remove assert --- .../main/java/org/elasticsearch/action/index/IndexRequest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 8f60146a5491e..a8d6220415a43 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -881,7 +881,6 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, Metadata metadata) { @Override public int route(IndexRouting indexRouting) { - assert routing == null : routing; return indexRouting.indexShard(id, routing, contentType, source, this::routing); } From 1930c59c27f27f74815664e2caf2786b40815832 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 7 Mar 2024 11:57:55 +0200 Subject: [PATCH 39/51] fix id test --- .../mapper/TsidExtractingIdFieldMapper.java | 23 ++++---- .../TsidExtractingIdFieldMapperTests.java | 58 ++++++++++--------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index ea5caae8cb7a9..a5516402755ef 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -70,18 +70,6 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra assert context.getDynamicMappers().isEmpty() == false || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); - - if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "_id must be unset or set to [%s] but was [%s] because [%s] is in time_series mode", - id, - context.sourceToParse().id(), - context.indexSettings().getIndexMetadata().getIndex().getName() - ) - ); - } } else if (context.sourceToParse().routing() != null) { int routingHash = TimeSeriesRoutingHashFieldMapper.decode(context.sourceToParse().routing()); id = createId(routingHash, tsid, timestamp); @@ -92,6 +80,17 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra + "] is in time_series mode" ); } + if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "_id must be unset or set to [%s] but was [%s] because [%s] is in time_series mode", + id, + context.sourceToParse().id(), + context.indexSettings().getIndexMetadata().getIndex().getName() + ) + ); + } context.id(id); BytesRef uidEncoded = Uid.encodeId(context.id()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index f40a3a8d621e5..6a7ac7dc3291a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -11,15 +11,14 @@ import org.apache.lucene.index.IndexableField; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.cluster.routing.IndexRouting; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.name.Named; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.ByteUtils; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; @@ -27,6 +26,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -35,8 +35,6 @@ public class TsidExtractingIdFieldMapperTests extends MetadataMapperTestCase { - private static final int SHARD_ID = 7; - private static class TestCase { private final String name; private final String expectedId; @@ -693,40 +691,50 @@ public TsidExtractingIdFieldMapperTests(@Named("testCase") TestCase testCase) { } public void testExpectedId() throws IOException { - assertThat( - parse(TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.source).id(), - equalTo(testCase.expectedId) - ); + assertThat(parse(mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); } public void testProvideExpectedId() throws IOException { assertThat(parse(testCase.expectedId, mapperService(), testCase.source).id(), equalTo(testCase.expectedId)); } + public void testProvideWrongId() { + String wrongId = testCase.expectedId + "wrong"; + Exception e = expectThrows(DocumentParsingException.class, () -> parse(wrongId, mapperService(), testCase.source)); + assertThat( + e.getCause().getMessage(), + equalTo( + "_id must be unset or set to [" + + testCase.expectedId + + "] but was [" + + testCase.expectedId + + "wrong] because [index] is in time_series mode" + ) + ); + } + public void testEquivalentSources() throws IOException { MapperService mapperService = mapperService(); for (CheckedConsumer equivalent : testCase.equivalentSources) { - assertThat( - parse(TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), mapperService, equivalent).id(), - equalTo(testCase.expectedId) - ); + assertThat(parse(mapperService, equivalent).id(), equalTo(testCase.expectedId)); } } - private ParsedDocument parse( - @Nullable String routingHash, - MapperService mapperService, - CheckedConsumer source - ) throws IOException { + private ParsedDocument parse(MapperService mapperService, CheckedConsumer source) throws IOException { + return parse(null, mapperService, source); + } + + private ParsedDocument parse(@Nullable String id, MapperService mapperService, CheckedConsumer source) + throws IOException { try (XContentBuilder builder = XContentBuilder.builder(randomFrom(XContentType.values()).xContent())) { builder.startObject(); source.accept(builder); builder.endObject(); SourceToParse sourceToParse = new SourceToParse( - null, + id, BytesReference.bytes(builder), builder.contentType(), - routingHash, + TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), Map.of(), DocumentSizeObserver.EMPTY_INSTANCE ); @@ -735,10 +743,8 @@ private ParsedDocument parse( } public void testRoutingPathCompliant() throws IOException { - IndexVersion version = IndexVersions.TIME_SERIES_ROUTING_HASH_IN_ID; - IndexRouting indexRouting = createIndexSettings(version, indexSettings(version)).getIndexRouting(); - assertThat(indexRouting.getShard(testCase.expectedId, null), equalTo(SHARD_ID)); - assertThat(indexRouting.deleteShard(testCase.expectedId, null), equalTo(SHARD_ID)); + byte[] bytes = Base64.getUrlDecoder().decode(testCase.expectedId); + assertEquals(ROUTING_HASH, ByteUtils.readIntLE(bytes, 0)); } private Settings indexSettings(IndexVersion version) { @@ -799,7 +805,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { public void testSourceDescription() throws IOException { assertThat(TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext()), equalTo("a time series document")); - ParsedDocument d = parse(TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.randomSource()); + ParsedDocument d = parse(mapperService(), testCase.randomSource()); IndexableField timestamp = d.rootDoc().getField(DataStreamTimestampFieldMapper.DEFAULT_PATH); assertThat( TsidExtractingIdFieldMapper.INSTANCE.documentDescription(documentParserContext(timestamp)), @@ -829,9 +835,7 @@ private TestDocumentParserContext documentParserContext(IndexableField... fields public void testParsedDescription() throws IOException { assertThat( - TsidExtractingIdFieldMapper.INSTANCE.documentDescription( - parse(TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), mapperService(), testCase.randomSource()) - ), + TsidExtractingIdFieldMapper.INSTANCE.documentDescription(parse(mapperService(), testCase.randomSource())), equalTo("[" + testCase.expectedId + "][" + testCase.expectedTsid + "@" + testCase.expectedTimestamp + "]") ); } From 5a0d8c942114da84a66912ee618f1333af0f9d9c Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 7 Mar 2024 12:48:58 +0200 Subject: [PATCH 40/51] fix painless --- .../action/PainlessExecuteAction.java | 8 +++++--- .../action/bulk/TransportShardBulkAction.java | 4 +--- .../index/codec/PerFieldMapperCodec.java | 2 +- .../index/engine/TranslogDirectoryReader.java | 11 +---------- .../index/mapper/SourceToParse.java | 4 ++++ .../elasticsearch/index/shard/IndexShard.java | 10 +--------- .../index/termvectors/TermVectorsService.java | 10 +--------- .../index/mapper/RoutingFieldMapperTests.java | 6 +----- .../mapper/TimeSeriesIdFieldMapperTests.java | 6 ++---- .../TsidExtractingIdFieldMapperTests.java | 6 +----- .../recovery/RecoverySourceHandlerTests.java | 19 +++++-------------- .../index/engine/TranslogHandler.java | 11 +---------- .../index/mapper/MapperServiceTestCase.java | 13 +------------ .../index/shard/IndexShardTestCase.java | 11 +---------- .../SourceOnlySnapshotShardTests.java | 11 +---------- 15 files changed, 27 insertions(+), 105 deletions(-) diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 874b2316406ef..66f3ee12d3219 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -61,6 +61,7 @@ import org.elasticsearch.index.mapper.OnScriptError; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.SearchExecutionContext; @@ -809,12 +810,13 @@ private static Response prepareRamIndex( BytesReference document = request.contextSetup.document; XContentType xContentType = request.contextSetup.xContentType; String id; + SourceToParse sourceToParse; if (indexService.getIndexSettings().getMode() == IndexMode.TIME_SERIES) { - id = null; // The id gets auto generated for time series indices. + sourceToParse = new SourceToParse(null, document, xContentType, TimeSeriesRoutingHashFieldMapper.encode(0)); } else { - id = "_id"; + sourceToParse = new SourceToParse("_id", document, xContentType); } - SourceToParse sourceToParse = new SourceToParse(id, document, xContentType); + DocumentMapper documentMapper = indexService.mapperService().documentMapper(); if (documentMapper == null) { documentMapper = DocumentMapper.createEmpty(indexService.mapperService()); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java index 70168f6a2b516..fe7af4bc26e6e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportShardBulkAction.java @@ -689,9 +689,7 @@ private static Engine.Result performOpOnReplica( indexRequest.id(), indexRequest.source(), indexRequest.getContentType(), - indexRequest.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE + indexRequest.routing() ); result = replica.applyIndexOperationOnReplica( primaryResponse.getSeqNo(), diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java index 397b7180e791b..ae497af887d9c 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java @@ -115,7 +115,7 @@ boolean useTSDBDocValuesFormat(final String field) { private boolean excludeFields(String fieldName) { // Avoid using tsdb codec for fields like _seq_no, _primary_term. - // But _tsid and _routing_path should always use the tsdb codec. + // But _tsid and _ts_routing_hash should always use the tsdb codec. return fieldName.startsWith("_") && fieldName.equals("_tsid") == false && fieldName.equals("_ts_routing_hash") == false; } diff --git a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java index e5eeac72927c0..e054fc52b562e 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java +++ b/server/src/main/java/org/elasticsearch/index/engine/TranslogDirectoryReader.java @@ -59,11 +59,9 @@ import org.elasticsearch.index.mapper.VersionFieldMapper; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.index.translog.Translog; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import java.io.IOException; import java.util.Collections; -import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -254,14 +252,7 @@ private LeafReader getDelegate() { private LeafReader createInMemoryLeafReader() { assert Thread.holdsLock(this); final ParsedDocument parsedDocs = documentParser.parseDocument( - new SourceToParse( - operation.id(), - operation.source(), - XContentHelper.xContentType(operation.source()), - operation.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - ), + new SourceToParse(operation.id(), operation.source(), XContentHelper.xContentType(operation.source()), operation.routing()), mappingLookup ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java index 12f74263a3bd7..6a020127019f5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceToParse.java @@ -52,6 +52,10 @@ public SourceToParse(String id, BytesReference source, XContentType xContentType this(id, source, xContentType, null, Map.of(), DocumentSizeObserver.EMPTY_INSTANCE); } + public SourceToParse(String id, BytesReference source, XContentType xContentType, String routing) { + this(id, source, xContentType, routing, Map.of(), DocumentSizeObserver.EMPTY_INSTANCE); + } + public BytesReference source() { return this.source; } diff --git a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java index 3bafe139756fd..046483a6b074f 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/elasticsearch/index/shard/IndexShard.java @@ -144,7 +144,6 @@ import org.elasticsearch.indices.recovery.RecoveryState; import org.elasticsearch.indices.recovery.RecoveryTarget; import org.elasticsearch.plugins.IndexStorePlugin; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.repositories.RepositoriesService; import org.elasticsearch.repositories.Repository; import org.elasticsearch.rest.RestStatus; @@ -1951,14 +1950,7 @@ private Engine.Result applyTranslogOperation(Engine engine, Translog.Operation o index.getAutoGeneratedIdTimestamp(), true, origin, - new SourceToParse( - index.id(), - index.source(), - XContentHelper.xContentType(index.source()), - index.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - ) + new SourceToParse(index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing()) ); } case DELETE -> { diff --git a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java index b4c0b200eb143..a30249e94177e 100644 --- a/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java +++ b/server/src/main/java/org/elasticsearch/index/termvectors/TermVectorsService.java @@ -41,7 +41,6 @@ import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentType; @@ -305,14 +304,7 @@ private static Fields generateTermVectors( } private static Fields generateTermVectorsFromDoc(IndexShard indexShard, TermVectorsRequest request) throws IOException { - SourceToParse source = new SourceToParse( - "_id_for_tv_api", - request.doc(), - request.xContentType(), - request.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - ); + SourceToParse source = new SourceToParse("_id_for_tv_api", request.doc(), request.xContentType(), request.routing()); DocumentParser documentParser = indexShard.mapperService().documentParser(); MappingLookup mappingLookup = indexShard.mapperService().mappingLookup(); ParsedDocument parsedDocument = documentParser.parseDocument(source, mappingLookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 53fcd3d331745..e0c092bfd0bfd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -12,7 +12,6 @@ import org.apache.lucene.search.IndexSearcher; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.index.query.SearchExecutionContext; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.Source; import org.elasticsearch.xcontent.XContentFactory; @@ -21,7 +20,6 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -53,9 +51,7 @@ public void testRoutingMapper() throws Exception { "1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field", "value").endObject()), XContentType.JSON, - "routing_value", - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE + "routing_value" ) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java index 7c689e88edf13..75f20e88d45cb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapperTests.java @@ -22,13 +22,11 @@ import org.elasticsearch.index.VersionType; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.shard.IndexShard; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import java.io.IOException; -import java.util.Map; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.hamcrest.Matchers.containsString; @@ -674,7 +672,7 @@ public void testParseWithDynamicMapping() { "@timestamp": 1609459200000, "dim": "6a841a21", "value": 100 - }"""), XContentType.JSON, TimeSeriesRoutingHashFieldMapper.encode(0), Map.of(), DocumentSizeObserver.EMPTY_INSTANCE); + }"""), XContentType.JSON, TimeSeriesRoutingHashFieldMapper.encode(0)); Engine.Index index = IndexShard.prepareIndex( mapper, source, @@ -703,7 +701,7 @@ public void testParseWithDynamicMappingInvalidRoutingHash() { "@timestamp": 1609459200000, "dim": "6a841a21", "value": 100 - }"""), XContentType.JSON, "no such routing hash", Map.of(), DocumentSizeObserver.EMPTY_INSTANCE); + }"""), XContentType.JSON, "no such routing hash"); var failure = expectThrows(DocumentParsingException.class, () -> { IndexShard.prepareIndex( mapper, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java index 6a7ac7dc3291a..78fcfdf2872ff 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapperTests.java @@ -19,7 +19,6 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; @@ -28,7 +27,6 @@ import java.util.ArrayList; import java.util.Base64; import java.util.List; -import java.util.Map; import java.util.stream.Stream; import static org.hamcrest.Matchers.equalTo; @@ -734,9 +732,7 @@ private ParsedDocument parse(@Nullable String id, MapperService mapperService, C id, BytesReference.bytes(builder), builder.contentType(), - TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE + TimeSeriesRoutingHashFieldMapper.encode(ROUTING_HASH) ); return mapperService.documentParser().parseDocument(sourceToParse, mapperService.mappingLookup()); } diff --git a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java index 44f57cf0768d4..6b23e2705bb7f 100644 --- a/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/indices/recovery/RecoverySourceHandlerTests.java @@ -73,7 +73,6 @@ import org.elasticsearch.indices.recovery.plan.PeerOnlyRecoveryPlannerService; import org.elasticsearch.indices.recovery.plan.RecoveryPlannerService; import org.elasticsearch.indices.recovery.plan.ShardRecoveryPlan; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.repositories.IndexId; import org.elasticsearch.test.CorruptionUtils; import org.elasticsearch.test.DummyShardLock; @@ -519,19 +518,11 @@ private TimeSeriesModeIndexOpFactory() throws IOException { @Override public Engine.Index createIndexOp(int docIdent) { - SourceToParse source = new SourceToParse( - null, - new BytesArray(Strings.format(""" - { - "@timestamp": %s, - "dim": "dim" - }""", docIdent)), - XContentType.JSON, - TimeSeriesRoutingHashFieldMapper.encode(0), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - - ); + SourceToParse source = new SourceToParse(null, new BytesArray(Strings.format(""" + { + "@timestamp": %s, + "dim": "dim" + }""", docIdent)), XContentType.JSON, TimeSeriesRoutingHashFieldMapper.encode(0)); return IndexShard.prepareIndex( mapper, source, diff --git a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java index 423990999fabd..d6e33c43e94c5 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/index/engine/TranslogHandler.java @@ -22,12 +22,10 @@ import org.elasticsearch.index.similarity.SimilarityService; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesModule; -import org.elasticsearch.plugins.internal.DocumentSizeObserver; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentParserConfiguration; import java.io.IOException; -import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import static java.util.Collections.emptyList; @@ -89,14 +87,7 @@ public Engine.Operation convertToEngineOp(Translog.Operation operation, Engine.O final Translog.Index index = (Translog.Index) operation; final Engine.Index engineIndex = IndexShard.prepareIndex( mapperService, - new SourceToParse( - index.id(), - index.source(), - XContentHelper.xContentType(index.source()), - index.routing(), - Map.of(), - DocumentSizeObserver.EMPTY_INSTANCE - ), + new SourceToParse(index.id(), index.source(), XContentHelper.xContentType(index.source()), index.routing()), index.seqNo(), index.primaryTerm(), index.version(), diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index c393042f07413..09c6eed08bf28 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -714,18 +714,7 @@ protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer Date: Thu, 7 Mar 2024 13:34:11 +0200 Subject: [PATCH 41/51] skip storing routing field --- .../org/elasticsearch/index/mapper/RoutingFieldMapper.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java index 3141b73174897..776e7a9033448 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RoutingFieldMapper.java @@ -11,6 +11,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.SearchExecutionContext; @@ -109,7 +110,8 @@ public boolean required() { @Override public void preParse(DocumentParserContext context) { String routing = context.sourceToParse().routing(); - if (routing != null) { + // TSDB uses field _ts_routing_hash instead. + if (routing != null && context.indexSettings().getMode() != IndexMode.TIME_SERIES) { context.doc().add(new StringField(fieldType().name(), routing, Field.Store.YES)); context.addToFieldNames(fieldType().name()); } From d35603cc9b6146d2536d5d1929697d3d17ec9469 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 7 Mar 2024 15:57:25 +0200 Subject: [PATCH 42/51] skip generating the id when available --- .../index/fieldvisitor/FieldsVisitor.java | 7 ++++++- .../index/mapper/TsidExtractingIdFieldMapper.java | 13 ++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 4789dcc131b89..9a4e467ef66bb 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -18,6 +18,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.Uid; import java.util.ArrayList; @@ -167,7 +168,11 @@ public String routing() { } List values = fieldsValues.get(RoutingFieldMapper.NAME); if (values == null || values.isEmpty()) { - return null; + // Check _ts_routing_hash for TSDB. + values = fieldsValues.get(TimeSeriesRoutingHashFieldMapper.NAME); + if (values == null || values.isEmpty()) { + return null; + } } assert values.size() == 1; return values.get(0).toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index a5516402755ef..dac566f2dbdcf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -74,11 +74,14 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra int routingHash = TimeSeriesRoutingHashFieldMapper.decode(context.sourceToParse().routing()); id = createId(routingHash, tsid, timestamp); } else { - throw new IllegalArgumentException( - "_ts_routing_hash was null but must be set because index [" - + context.indexSettings().getIndexMetadata().getIndex().getName() - + "] is in time_series mode" - ); + if (context.sourceToParse().id() == null) { + throw new IllegalArgumentException( + "_ts_routing_hash was null but must be set because index [" + + context.indexSettings().getIndexMetadata().getIndex().getName() + + "] is in time_series mode" + ); + } + id = context.sourceToParse().id(); } if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { throw new IllegalArgumentException( From 52030686eddf2498664fc304495fbae7414ee00e Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 7 Mar 2024 16:50:19 +0200 Subject: [PATCH 43/51] skip assert on routing in TranslogWriter --- .../elasticsearch/index/fieldvisitor/FieldsVisitor.java | 7 +------ .../org/elasticsearch/index/translog/TranslogWriter.java | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java index 9a4e467ef66bb..4789dcc131b89 100644 --- a/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java +++ b/server/src/main/java/org/elasticsearch/index/fieldvisitor/FieldsVisitor.java @@ -18,7 +18,6 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.index.mapper.TimeSeriesRoutingHashFieldMapper; import org.elasticsearch.index.mapper.Uid; import java.util.ArrayList; @@ -168,11 +167,7 @@ public String routing() { } List values = fieldsValues.get(RoutingFieldMapper.NAME); if (values == null || values.isEmpty()) { - // Check _ts_routing_hash for TSDB. - values = fieldsValues.get(TimeSeriesRoutingHashFieldMapper.NAME); - if (values == null || values.isEmpty()) { - return null; - } + return null; } assert values.size() == 1; return values.get(0).toString(); diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java index 89d1314134387..d519119978964 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java @@ -276,7 +276,6 @@ private synchronized boolean assertNoSeqNumberConflict(long seqNo, BytesReferenc if (newOp instanceof final Translog.Index o2 && prvOp instanceof final Translog.Index o1) { sameOp = Objects.equals(o1.id(), o2.id()) && Objects.equals(o1.source(), o2.source()) - && Objects.equals(o1.routing(), o2.routing()) && o1.primaryTerm() == o2.primaryTerm() && o1.seqNo() == o2.seqNo() && o1.version() == o2.version(); From 4646c8cf951b5c5dcf09ab15a0f493a2a792f38f Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Thu, 7 Mar 2024 17:49:26 +0200 Subject: [PATCH 44/51] skip id mismatch assert --- .../mapper/TsidExtractingIdFieldMapper.java | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index dac566f2dbdcf..3241a65de6885 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -70,6 +70,17 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra assert context.getDynamicMappers().isEmpty() == false || context.getDynamicRuntimeFields().isEmpty() == false || id.equals(indexRouting.createId(context.sourceToParse().getXContentType(), context.sourceToParse().source(), suffix)); + if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "_id must be unset or set to [%s] but was [%s] because [%s] is in time_series mode", + id, + context.sourceToParse().id(), + context.indexSettings().getIndexMetadata().getIndex().getName() + ) + ); + } } else if (context.sourceToParse().routing() != null) { int routingHash = TimeSeriesRoutingHashFieldMapper.decode(context.sourceToParse().routing()); id = createId(routingHash, tsid, timestamp); @@ -81,19 +92,9 @@ public static void createField(DocumentParserContext context, IndexRouting.Extra + "] is in time_series mode" ); } + // In Translog operations, the id has already been generated based on the routing hash while the latter is no longer. id = context.sourceToParse().id(); } - if (context.sourceToParse().id() != null && false == context.sourceToParse().id().equals(id)) { - throw new IllegalArgumentException( - String.format( - Locale.ROOT, - "_id must be unset or set to [%s] but was [%s] because [%s] is in time_series mode", - id, - context.sourceToParse().id(), - context.indexSettings().getIndexMetadata().getIndex().getName() - ) - ); - } context.id(id); BytesRef uidEncoded = Uid.encodeId(context.id()); From 513e4265d225c4606b7474937b12bf623f347c71 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 13 Mar 2024 17:54:47 +0200 Subject: [PATCH 45/51] changelog update --- docs/changelog/105501.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/105501.yaml b/docs/changelog/105501.yaml index 1bf658529af58..50c903d5cc518 100644 --- a/docs/changelog/105501.yaml +++ b/docs/changelog/105501.yaml @@ -1,5 +1,5 @@ pr: 105501 -summary: Introduce TimeSeriesRoutingIdFieldMapper to support non-keyword dimensions +summary: Support non-keyword dimensions in TSDB area: TSDB type: enhancement issues: [] From bd8f7b959b665a88601f849e996bcd343915020a Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 13 Mar 2024 17:59:28 +0200 Subject: [PATCH 46/51] revert redundant changes --- .../rest-api-spec/test/update_by_query/90_tsdb.yml | 12 ++++++------ .../elasticsearch/index/translog/TranslogWriter.java | 1 + .../replication/ESIndexLevelReplicationTestCase.java | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml index 2c31847391cc2..cee4395eb87f8 100644 --- a/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml +++ b/modules/reindex/src/yamlRestTest/resources/rest-api-spec/test/update_by_query/90_tsdb.yml @@ -77,7 +77,7 @@ update tag field: update dimension field: # TODO better error message - do: - catch: conflict + catch: bad_request update_by_query: index: tsdb body: @@ -86,17 +86,17 @@ update dimension field: source: ctx._source.k8s.pod.uid = "12342134" - match: {updated: 0} - - match: {version_conflicts: 1} + - match: {version_conflicts: 0} - match: {batches: 1} - match: {throttled_millis: 0} - gte: { took: 0 } - - match: {failures.0.cause.reason: /version\ conflict/} + - match: {failures.0.cause.caused_by.reason: /_id\ must\ be\ unset\ or\ set\ to\ .+/} --- update timestamp: # TODO better error message - do: - catch: conflict + catch: bad_request update_by_query: index: tsdb body: @@ -105,8 +105,8 @@ update timestamp: source: ctx._source["@timestamp"] = "2021-04-28T18:50:33.142Z" - match: {updated: 0} - - match: {version_conflicts: 1} + - match: {version_conflicts: 0} - match: {batches: 1} - match: {throttled_millis: 0} - gte: { took: 0 } - - match: {failures.0.cause.reason: /version\ conflict/} + - match: {failures.0.cause.caused_by.reason: /_id\ must\ be\ unset\ or\ set\ to\ .+/} diff --git a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java index d519119978964..89d1314134387 100644 --- a/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java +++ b/server/src/main/java/org/elasticsearch/index/translog/TranslogWriter.java @@ -276,6 +276,7 @@ private synchronized boolean assertNoSeqNumberConflict(long seqNo, BytesReferenc if (newOp instanceof final Translog.Index o2 && prvOp instanceof final Translog.Index o1) { sameOp = Objects.equals(o1.id(), o2.id()) && Objects.equals(o1.source(), o2.source()) + && Objects.equals(o1.routing(), o2.routing()) && o1.primaryTerm() == o2.primaryTerm() && o1.seqNo() == o2.seqNo() && o1.version() == o2.version(); diff --git a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java index 8ac0e4952c55e..ba6d7e441ef4a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/replication/ESIndexLevelReplicationTestCase.java @@ -872,7 +872,7 @@ private void executeShardBulkOnPrimary( ) { for (BulkItemRequest itemRequest : request.items()) { if (itemRequest.request() instanceof IndexRequest) { - itemRequest.request().process(primary.indexSettings().getIndexRouting()); + ((IndexRequest) itemRequest.request()).process(primary.indexSettings().getIndexRouting()); } } final PlainActionFuture permitAcquiredFuture = new PlainActionFuture<>(); From ada4293fbde8a63d1c6a981ab484f93304832512 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 20 Mar 2024 16:49:12 +0200 Subject: [PATCH 47/51] remove supportsDimension --- docs/changelog/105501.yaml | 2 +- .../rest-api-spec/test/tsdb/20_mapping.yml | 2 +- .../index/mapper/IpFieldMapper.java | 5 ----- .../index/mapper/KeywordFieldMapper.java | 5 ----- .../index/mapper/MappedFieldType.java | 20 ------------------- .../index/mapper/NumberFieldMapper.java | 5 ----- .../flattened/FlattenedFieldMapper.java | 10 ---------- .../index/mapper/MapperTestCase.java | 1 - .../unsignedlong/UnsignedLongFieldMapper.java | 5 ----- 9 files changed, 2 insertions(+), 53 deletions(-) diff --git a/docs/changelog/105501.yaml b/docs/changelog/105501.yaml index 50c903d5cc518..2e5e375764640 100644 --- a/docs/changelog/105501.yaml +++ b/docs/changelog/105501.yaml @@ -1,5 +1,5 @@ pr: 105501 -summary: Support non-keyword dimensions in TSDB +summary: Support non-keyword dimensions as routing fields in TSDB area: TSDB type: enhancement issues: [] diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml index 8ccac3f95a992..807f60a0faf35 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml @@ -158,7 +158,7 @@ non keyword matches routing_path: reason: routing_path error message updated in 8.14.0 - do: - catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was \[date\]./' + catch: '/All fields that match routing_path must be configured with \[time_series_dimension: true\] or flattened fields with a list of dimensions in \[time_series_dimensions\] and without the \[script\] parameter. \[@timestamp\] was not a dimension./' indices.create: index: test_index body: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index d9051e69e34f7..3ef5079c30e58 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -251,11 +251,6 @@ public boolean isDimension() { return isDimension; } - @Override - public boolean supportsDimension() { - return true; - } - @Override public boolean hasScriptValues() { return scriptValues != null; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e33f8b593d80d..02a11beecaef4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -816,11 +816,6 @@ public boolean isDimension() { return isDimension; } - @Override - public boolean supportsDimension() { - return true; - } - @Override public boolean hasScriptValues() { return scriptValues != null; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index c3cbf47c25c1a..bdbb4910eb4ba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -195,13 +195,6 @@ public boolean isDimension() { return false; } - /** - * @return true if field can be marked as a dimension field - */ - public boolean supportsDimension() { - return false; - } - /** * @return true if field has script values. */ @@ -635,19 +628,6 @@ public TermsEnum getTerms(IndexReader reader, String prefix, boolean caseInsensi * Validate that this field can be the target of {@link IndexMetadata#INDEX_ROUTING_PATH}. */ public void validateMatchedRoutingPath(String routingPath) { - if (supportsDimension() == false) { - throw new IllegalArgumentException( - "All fields that match routing_path " - + "must be configured with [time_series_dimension: true] " - + "or flattened fields with a list of dimensions in [time_series_dimensions] and " - + "without the [script] parameter. [" - + name() - + "] was [" - + typeName() - + "]." - ); - } - if (hasScriptValues()) { throw new IllegalArgumentException( "All fields that match routing_path must be configured with [time_series_dimension: true] " diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 79419548a95c2..c7fd78d3a2aba 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1744,11 +1744,6 @@ public boolean isDimension() { return isDimension; } - @Override - public boolean supportsDimension() { - return true; - } - @Override public boolean hasScriptValues() { return scriptValues != null; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index c502371db6dea..8feaba73b1dd4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -236,11 +236,6 @@ public boolean isDimension() { return isDimension; } - @Override - public boolean supportsDimension() { - return true; - } - KeyedFlattenedFieldType( String rootName, boolean indexed, @@ -722,11 +717,6 @@ public boolean isDimension() { return isDimension; } - @Override - public boolean supportsDimension() { - return true; - } - @Override public List dimensions() { return this.dimensions; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 4bc9b10f1a764..23931f3ef33ea 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1032,7 +1032,6 @@ public final void testMinimalIsInvalidInRoutingPath() throws IOException { private String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { if (mapper instanceof FieldMapper fieldMapper - && fieldMapper.fieldType().supportsDimension() && fieldMapper.fieldType().isDimension() == false) { return "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 7c7b30f0bf7f3..e8bb148315831 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -544,11 +544,6 @@ public boolean isDimension() { return isDimension; } - @Override - public boolean supportsDimension() { - return true; - } - /** * If field is a time series metric field, returns its metric type * @return the metric type or null From 3a5d2342b900b211d111c0c8aa07354a70cdd1f8 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 20 Mar 2024 17:08:39 +0200 Subject: [PATCH 48/51] spotless --- .../java/org/elasticsearch/index/mapper/MapperTestCase.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 23931f3ef33ea..fa0f0e1b95f54 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1031,8 +1031,7 @@ public final void testMinimalIsInvalidInRoutingPath() throws IOException { } private String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { - if (mapper instanceof FieldMapper fieldMapper - && fieldMapper.fieldType().isDimension() == false) { + if (mapper instanceof FieldMapper fieldMapper && fieldMapper.fieldType().isDimension() == false) { return "All fields that match routing_path must be configured with [time_series_dimension: true] " + "or flattened fields with a list of dimensions in [time_series_dimensions] and " + "without the [script] parameter. [" From 6f56b4d0ca226d04d98ded94153166cffed5f87b Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 20 Mar 2024 17:42:35 +0200 Subject: [PATCH 49/51] rename var for forcing dimension --- .../java/org/elasticsearch/index/mapper/FieldMapper.java | 9 ++++++--- .../org/elasticsearch/index/mapper/IpFieldMapper.java | 2 +- .../elasticsearch/index/mapper/KeywordFieldMapper.java | 2 +- .../elasticsearch/index/mapper/NumberFieldMapper.java | 2 +- .../index/mapper/PassThroughObjectMapper.java | 2 +- .../xpack/unsignedlong/UnsignedLongFieldMapper.java | 2 +- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index c1423499b8c75..94397b452ffc8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1415,16 +1415,19 @@ private static boolean isDeprecatedParameter(String propName, IndexVersion index } } + /** + * Creates mappers for fields that can act as time-series dimensions. + */ public abstract static class DimensionBuilder extends Builder { - protected boolean isDimension = false; + protected boolean inheritDimensionParameterFromParentObject = false; public DimensionBuilder(String name) { super(name); } - void setDimension() { - this.isDimension = true; + void setInheritDimensionParameterFromParentObject() { + this.inheritDimensionParameterFromParentObject = true; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 3ef5079c30e58..ceb0ec3105895 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -166,7 +166,7 @@ protected Parameter[] getParameters() { @Override public IpFieldMapper build(MapperBuilderContext context) { - if (super.isDimension || context.parentObjectContainsDimensions()) { + if (super.inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions()) { dimension.setValue(true); } return new IpFieldMapper( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 02a11beecaef4..e365b9b713511 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -304,7 +304,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } - if (super.isDimension || context.parentObjectContainsDimensions()) { + if (super.inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions()) { dimension(true); } return new KeywordFieldType( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index c7fd78d3a2aba..27dc378406909 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -261,7 +261,7 @@ protected Parameter[] getParameters() { @Override public NumberFieldMapper build(MapperBuilderContext context) { - if (super.isDimension || context.parentObjectContainsDimensions()) { + if (super.inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions()) { dimension.setValue(true); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java index ed22b0f53a1e7..16b4d0b49917f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PassThroughObjectMapper.java @@ -42,7 +42,7 @@ public Builder(String name) { @Override public PassThroughObjectMapper.Builder add(Mapper.Builder builder) { if (timeSeriesDimensionSubFields.value() && builder instanceof FieldMapper.DimensionBuilder dimensionBuilder) { - dimensionBuilder.setDimension(); + dimensionBuilder.setInheritDimensionParameterFromParentObject(); } super.add(builder); return this; diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index e8bb148315831..7128c65b28f96 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -195,7 +195,7 @@ Number parsedNullValue() { @Override public UnsignedLongFieldMapper build(MapperBuilderContext context) { - if (super.isDimension || context.parentObjectContainsDimensions()) { + if (super.inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions()) { dimension.setValue(true); } UnsignedLongFieldType fieldType = new UnsignedLongFieldType( From 14b162d2e6e04f116e0998dc3196a4d4753945f8 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 20 Mar 2024 17:54:32 +0200 Subject: [PATCH 50/51] small refactor --- .../java/org/elasticsearch/index/mapper/FieldMapper.java | 6 +++++- .../java/org/elasticsearch/index/mapper/IpFieldMapper.java | 2 +- .../org/elasticsearch/index/mapper/KeywordFieldMapper.java | 2 +- .../org/elasticsearch/index/mapper/NumberFieldMapper.java | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 94397b452ffc8..2e4bdc4231b5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1420,7 +1420,7 @@ private static boolean isDeprecatedParameter(String propName, IndexVersion index */ public abstract static class DimensionBuilder extends Builder { - protected boolean inheritDimensionParameterFromParentObject = false; + private boolean inheritDimensionParameterFromParentObject = false; public DimensionBuilder(String name) { super(name); @@ -1429,6 +1429,10 @@ public DimensionBuilder(String name) { void setInheritDimensionParameterFromParentObject() { this.inheritDimensionParameterFromParentObject = true; } + + boolean inheritDimensionParameterFromParentObject(MapperBuilderContext context) { + return inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions(); + } } public static BiConsumer notInMultiFields(String type) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index ceb0ec3105895..2e0fc68770045 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -166,7 +166,7 @@ protected Parameter[] getParameters() { @Override public IpFieldMapper build(MapperBuilderContext context) { - if (super.inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions()) { + if (inheritDimensionParameterFromParentObject(context)) { dimension.setValue(true); } return new IpFieldMapper( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e365b9b713511..4024798a85370 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -304,7 +304,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType } else if (splitQueriesOnWhitespace.getValue()) { searchAnalyzer = Lucene.WHITESPACE_ANALYZER; } - if (super.inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions()) { + if (inheritDimensionParameterFromParentObject(context)) { dimension(true); } return new KeywordFieldType( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 27dc378406909..c04c3e5afdc70 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -261,7 +261,7 @@ protected Parameter[] getParameters() { @Override public NumberFieldMapper build(MapperBuilderContext context) { - if (super.inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions()) { + if (inheritDimensionParameterFromParentObject(context)) { dimension.setValue(true); } From d9eb1290e756fb0f5d7930635654ac6680d00f95 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 20 Mar 2024 18:05:14 +0200 Subject: [PATCH 51/51] small refactor --- .../main/java/org/elasticsearch/index/mapper/FieldMapper.java | 2 +- .../xpack/unsignedlong/UnsignedLongFieldMapper.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 2e4bdc4231b5a..e029aaa657d23 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1430,7 +1430,7 @@ void setInheritDimensionParameterFromParentObject() { this.inheritDimensionParameterFromParentObject = true; } - boolean inheritDimensionParameterFromParentObject(MapperBuilderContext context) { + protected boolean inheritDimensionParameterFromParentObject(MapperBuilderContext context) { return inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions(); } } diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 7128c65b28f96..e0ce1f92b2a37 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -195,7 +195,7 @@ Number parsedNullValue() { @Override public UnsignedLongFieldMapper build(MapperBuilderContext context) { - if (super.inheritDimensionParameterFromParentObject || context.parentObjectContainsDimensions()) { + if (inheritDimensionParameterFromParentObject(context)) { dimension.setValue(true); } UnsignedLongFieldType fieldType = new UnsignedLongFieldType(