From b012559caf008d9be607b548c55943f80dc77ee0 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 16 Dec 2020 09:51:51 +0100 Subject: [PATCH 1/2] Fix search_as_you_type field with term_vector This commit fixes a bug in the search_as_you_type field that was introduced during the refactoring of the field mapper. The prefix field that is used internally by the search_as_you_type mapper doesn't need term vector even if they are activated on the main field. So this commit ensures that we don't copy the options from the main field when we create the prefix sub-field. Closes #66407 --- .../mapper/SearchAsYouTypeFieldMapper.java | 13 ++--- .../SearchAsYouTypeFieldMapperTests.java | 47 +++++++++++++++---- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index d4e236003696..46d327474a91 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -166,8 +166,8 @@ public SearchAsYouTypeFieldMapper build(ContentPath contentPath) { indexAnalyzers.put(ft.name(), indexAnalyzer); // set up the prefix field - FieldType prefixft = new FieldType(fieldType); - prefixft.setStoreTermVectors(false); + FieldType prefixft = new FieldType(); + prefixft.setIndexOptions(fieldType.indexOptions()); prefixft.setOmitNorms(true); prefixft.setStored(false); final String fullName = buildFullName(contentPath); @@ -179,10 +179,8 @@ public SearchAsYouTypeFieldMapper build(ContentPath contentPath) { SearchAsYouTypeAnalyzer.withShingle(searchAnalyzer.analyzer(), maxShingleSize.getValue())); // don't wrap the root field's search quote analyzer as prefix field doesn't support phrase queries TextSearchInfo prefixSearchInfo = new TextSearchInfo(prefixft, similarity.getValue(), prefixSearchWrapper, searchAnalyzer); - final PrefixFieldType prefixFieldType - = new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM); - final NamedAnalyzer prefixAnalyzer - = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper); + final PrefixFieldType prefixFieldType = new PrefixFieldType(fullName, prefixSearchInfo, Defaults.MIN_GRAM, Defaults.MAX_GRAM); + final NamedAnalyzer prefixAnalyzer = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, prefixIndexWrapper); final PrefixFieldMapper prefixFieldMapper = new PrefixFieldMapper(prefixft, prefixFieldType); indexAnalyzers.put(prefixFieldType.name(), prefixAnalyzer); @@ -206,8 +204,7 @@ public SearchAsYouTypeFieldMapper build(ContentPath contentPath) { final ShingleFieldType shingleFieldType = new ShingleFieldType(fieldName, shingleSize, textSearchInfo); shingleFieldType.setPrefixFieldType(prefixFieldType); shingleFieldTypes[i] = shingleFieldType; - NamedAnalyzer shingleAnalyzer - = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper); + NamedAnalyzer shingleAnalyzer = new NamedAnalyzer(indexAnalyzer.name(), AnalyzerScope.INDEX, shingleIndexWrapper); shingleFieldMappers[i] = new ShingleFieldMapper(shingleft, shingleFieldType); indexAnalyzers.put(shingleFieldType.name(), shingleAnalyzer); } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index fa331fee2eff..9e6e2223637b 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -328,18 +328,47 @@ public void testStoredOnly() throws IOException { } public void testTermVectors() throws IOException { - DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type").field("term_vector", "yes"))); - ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); + for (String termVector : new String[] { "yes", "with_positions", "with_offsets", "with_positions_offsets", "with_positions_payloads", "with_positions_offsets_payloads"}) { + DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type") + .field("term_vector", termVector))); + ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text"))); - assertTrue(fieldType(doc, "field").storeTermVectors()); + IndexableFieldType rootField = fieldType(doc, "field"); + assertTrue(rootField.storeTermVectors()); + if (termVector.contains("positions")) { + assertThat(rootField.storeTermVectorPositions(), equalTo(termVector.contains("positions"))); + } + if (termVector.contains("offsets")) { + assertTrue(rootField.storeTermVectorOffsets()); + assertThat(rootField.storeTermVectorOffsets(), equalTo(termVector.contains("offsets"))); + } + if (termVector.contains("payloads")) { + assertTrue(rootField.storeTermVectorPayloads()); + assertThat(rootField.storeTermVectorPayloads(), equalTo(termVector.contains("payloads"))); + } - Stream.of( - fieldType(doc, "field._2gram"), - fieldType(doc, "field._3gram") - ).forEach(ft -> assertTrue(ft.storeTermVectors())); + Stream.of( + fieldType(doc, "field._2gram"), + fieldType(doc, "field._3gram") + ).forEach(ft -> { + assertTrue(ft.storeTermVectors()); + if (termVector.contains("positions")) { + assertThat(ft.storeTermVectorPositions(), equalTo(termVector.contains("positions"))); + } + if (termVector.contains("offsets")) { + assertThat(ft.storeTermVectorOffsets(), equalTo(termVector.contains("offsets"))); + } + if (termVector.contains("payloads")) { + assertThat(ft.storeTermVectorPayloads(), equalTo(termVector.contains("payloads"))); + } + }); - PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(mapper, "field._index_prefix"); - assertFalse(prefixFieldMapper.fieldType.storeTermVectors()); + PrefixFieldMapper prefixFieldMapper = getPrefixFieldMapper(mapper, "field._index_prefix"); + assertFalse(prefixFieldMapper.fieldType.storeTermVectors()); + assertFalse(prefixFieldMapper.fieldType.storeTermVectorOffsets()); + assertFalse(prefixFieldMapper.fieldType.storeTermVectorPositions()); + assertFalse(prefixFieldMapper.fieldType.storeTermVectorPayloads()); + } } public void testNorms() throws IOException { From 1b082362e4c2d4587dd4950970d5f7dff2f34854 Mon Sep 17 00:00:00 2001 From: jimczi Date: Wed, 16 Dec 2020 10:03:53 +0100 Subject: [PATCH 2/2] line len --- .../index/mapper/SearchAsYouTypeFieldMapperTests.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java index 9e6e2223637b..4c154c1fc531 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapperTests.java @@ -328,7 +328,8 @@ public void testStoredOnly() throws IOException { } public void testTermVectors() throws IOException { - for (String termVector : new String[] { "yes", "with_positions", "with_offsets", "with_positions_offsets", "with_positions_payloads", "with_positions_offsets_payloads"}) { + for (String termVector : new String[] { "yes", "with_positions", "with_offsets", "with_positions_offsets", + "with_positions_payloads", "with_positions_offsets_payloads"}) { DocumentMapper mapper = createDocumentMapper(fieldMapping(b -> b.field("type", "search_as_you_type") .field("term_vector", termVector))); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "some text")));