From 96dd66d2c944826b6d12781e01fe1cd266cf5018 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 16 Oct 2024 13:09:43 +0300 Subject: [PATCH 1/4] Reset array scope tracking for nested objects --- muted-tests.yml | 3 --- .../org/elasticsearch/index/mapper/DocumentParserContext.java | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/muted-tests.yml b/muted-tests.yml index 0b24bac278fa6..910f535108017 100644 --- a/muted-tests.yml +++ b/muted-tests.yml @@ -382,9 +382,6 @@ tests: issue: https://github.com/elastic/elasticsearch/issues/114839 - class: org.elasticsearch.license.LicensingTests issue: https://github.com/elastic/elasticsearch/issues/114865 -- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT - method: testTermsQuery - issue: https://github.com/elastic/elasticsearch/issues/114873 - class: org.elasticsearch.xpack.enrich.EnrichIT method: testDeleteIsCaseSensitive issue: https://github.com/elastic/elasticsearch/issues/114840 diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 2eec14bd1a8d6..70e31f5bc7878 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -709,12 +709,14 @@ public final DocumentParserContext createNestedContext(NestedObjectMapper nested * Return a new context that has the provided document as the current document. */ public final DocumentParserContext switchDoc(final LuceneDocument document) { - return new Wrapper(this.parent, this) { + DocumentParserContext cloned = new Wrapper(this.parent, this) { @Override public LuceneDocument doc() { return document; } }; + cloned.inArrayScope = false; // Reset for new document. + return cloned; } /** From c34b66d4a6a2dfac0d763f76c62f26d41ab22637 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 16 Oct 2024 14:15:00 +0300 Subject: [PATCH 2/4] update --- .../elasticsearch/index/mapper/DocumentParserContext.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 70e31f5bc7878..f5fcd1f6f0890 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -111,7 +111,7 @@ public int get() { private final Set ignoredFields; private final List ignoredFieldValues; private final List ignoredFieldsMissingValues; - private final boolean inArrayScopeEnabled; + private boolean inArrayScopeEnabled; private boolean inArrayScope; private final Map> dynamicMappers; @@ -715,7 +715,11 @@ public LuceneDocument doc() { return document; } }; - cloned.inArrayScope = false; // Reset for new document. + // Disable tracking array scopes for ignored source, as it would be added to the parent doc. + // Nested documents are added to preserve object structure within arrays of objects, so the use + // of ignored source inside them should be mostly redundant. + cloned.inArrayScope = false; + cloned.inArrayScopeEnabled = false; return cloned; } From e7108ae5d02902736a1e72446e52ff614f4bfbee Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 16 Oct 2024 15:00:44 +0300 Subject: [PATCH 3/4] update --- .../index/mapper/DocumentParserContext.java | 15 +++++----- .../mapper/IgnoredSourceFieldMapperTests.java | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index f5fcd1f6f0890..55926b3d0539f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -376,13 +376,14 @@ public final Collection getIgnoredFieldsMiss * Applies to synthetic source only. */ public final DocumentParserContext maybeCloneForArray(Mapper mapper) throws IOException { - if (canAddIgnoredField() && mapper instanceof ObjectMapper && inArrayScopeEnabled) { - boolean isNested = mapper instanceof NestedObjectMapper; - if ((inArrayScope == false && isNested == false) || (inArrayScope && isNested)) { - DocumentParserContext subcontext = switchParser(parser()); - subcontext.inArrayScope = inArrayScope == false; - return subcontext; - } + if (canAddIgnoredField() + && mapper instanceof ObjectMapper + && mapper instanceof NestedObjectMapper == false + && inArrayScope == false + && inArrayScopeEnabled) { + DocumentParserContext subcontext = switchParser(parser()); + subcontext.inArrayScope = true; + return subcontext; } return this; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java index 5eac5acdca286..934744ef3ef96 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java @@ -932,6 +932,36 @@ public void testConflictingFieldNameAfterArray() throws IOException { {"path":{"id":0.1,"to":{"id":[1,20,3,10]}}}""", syntheticSource); } + public void testArrayWithNestedObjects() throws IOException { + DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { + b.startObject("path").startObject("properties"); + { + b.startObject("to").field("type", "nested").startObject("properties"); + { + b.startObject("id").field("type", "integer").field("synthetic_source_keep", "arrays").endObject(); + } + b.endObject().endObject(); + } + b.endObject().endObject(); + })).documentMapper(); + + var syntheticSource = syntheticSource(documentMapper, b -> { + b.startArray("path"); + { + b.startObject().startArray("to"); + { + b.startObject().array("id", 1, 20, 3).endObject(); + b.startObject().field("id", 10).endObject(); + } + b.endArray().endObject(); + b.startObject().startObject("to").field("id", "0.1").endObject().endObject(); + } + b.endArray(); + }); + assertEquals(""" + {"path":{"to":[{"id":[1,20,3]},{"id":10},{"id":0}]}}""", syntheticSource); + } + public void testArrayWithinArray() throws IOException { DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> { b.startObject("path"); From a5a2412f5353e77533395ab0eede4e0ebc2aeab9 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas Date: Wed, 16 Oct 2024 15:04:25 +0300 Subject: [PATCH 4/4] update --- .../org/elasticsearch/index/mapper/DocumentParserContext.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index 55926b3d0539f..ef87ce52fbabf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -718,7 +718,7 @@ public LuceneDocument doc() { }; // Disable tracking array scopes for ignored source, as it would be added to the parent doc. // Nested documents are added to preserve object structure within arrays of objects, so the use - // of ignored source inside them should be mostly redundant. + // of ignored source for arrays inside them should be mostly redundant. cloned.inArrayScope = false; cloned.inArrayScopeEnabled = false; return cloned;