From df6fcff44d50097b1a0306e9b42c726b5aa6e7bc Mon Sep 17 00:00:00 2001 From: Huaixinww Date: Mon, 8 Jan 2024 11:22:15 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96term=E3=80=81match=E6=9F=A5?= =?UTF-8?q?=E8=AF=A2=E5=AF=B9object=E5=AF=B9=E8=B1=A1fieldname=E7=9A=84?= =?UTF-8?q?=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../search/HavenaskSearchQueryProcessor.java | 26 +++++--- .../HavenaskSearchQueryProcessorTests.java | 63 ++++++++++++++----- 2 files changed, 68 insertions(+), 21 deletions(-) diff --git a/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/search/HavenaskSearchQueryProcessor.java b/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/search/HavenaskSearchQueryProcessor.java index d67fdb3a..ac1da310 100644 --- a/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/search/HavenaskSearchQueryProcessor.java +++ b/elastic-fed/modules/havenask-engine/src/main/java/org/havenask/engine/search/HavenaskSearchQueryProcessor.java @@ -146,7 +146,7 @@ public String transferSearchRequest2HavenaskSql(String table, SearchSourceBuilde checkVectorMagnitude(similarity, proximaQueryBuilder.getVector()); selectParams.append(", ").append(getScoreComputeStr(fieldName, similarity)).append(" as _score"); - where.append(" where MATCHINDEX('").append(proximaQueryBuilder.getFieldName()).append("', '"); + where.append(" where MATCHINDEX('").append(fieldName).append("', '"); for (int i = 0; i < proximaQueryBuilder.getVector().length; i++) { where.append(proximaQueryBuilder.getVector()[i]); @@ -158,11 +158,15 @@ public String transferSearchRequest2HavenaskSql(String table, SearchSourceBuilde orderBy.append(" order by _score desc"); } else if (queryBuilder instanceof TermQueryBuilder) { TermQueryBuilder termQueryBuilder = (TermQueryBuilder) queryBuilder; - where.append(" where ").append(termQueryBuilder.fieldName()).append("='").append(termQueryBuilder.value()).append("'"); + where.append(" where ") + .append(Schema.encodeFieldWithDot(termQueryBuilder.fieldName())) + .append("='") + .append(termQueryBuilder.value()) + .append("'"); } else if (queryBuilder instanceof MatchQueryBuilder) { MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) queryBuilder; where.append(" where MATCHINDEX('") - .append(matchQueryBuilder.fieldName()) + .append(Schema.encodeFieldWithDot(matchQueryBuilder.fieldName())) .append("', '") .append(matchQueryBuilder.value()) .append("')"); @@ -170,10 +174,18 @@ public String transferSearchRequest2HavenaskSql(String table, SearchSourceBuilde throw new IOException("unsupported DSL: " + dsl); } } - sqlQuery.append("select").append(selectParams).append(" from ").append('`').append(table).append('`'); - sqlQuery.append(where).append(orderBy); - - sqlQuery.append(" limit ").append(size).append(" offset ").append(from); + sqlQuery.append("select") + .append(selectParams) + .append(" from ") + .append('`') + .append(table) + .append('`') + .append(where) + .append(orderBy) + .append(" limit ") + .append(size) + .append(" offset ") + .append(from); return sqlQuery.toString(); } diff --git a/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/search/fetch/HavenaskSearchQueryProcessorTests.java b/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/search/fetch/HavenaskSearchQueryProcessorTests.java index 9ac34405..98441619 100644 --- a/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/search/fetch/HavenaskSearchQueryProcessorTests.java +++ b/elastic-fed/modules/havenask-engine/src/test/java/org/havenask/engine/search/fetch/HavenaskSearchQueryProcessorTests.java @@ -32,6 +32,7 @@ public class HavenaskSearchQueryProcessorTests extends HavenaskTestCase { private QrsClient qrsClient = mock(QrsClient.class); + private HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); private Map indexMapping = new HashMap<>(); private Map ObjectMapping = new HashMap<>(); @@ -56,9 +57,13 @@ public void setup() { Map userMapping = new HashMap<>(); Map userPropertiesMapping = new HashMap<>(); Map userImageMapping = new HashMap<>(); + Map userFirstNameMapping = new HashMap<>(); userImageMapping.put("type", "vector"); userImageMapping.put("similarity", "L2_NORM"); userPropertiesMapping.put("image_vector", userImageMapping); + userFirstNameMapping.put("type", "keyword"); + userPropertiesMapping.put("first_name", userFirstNameMapping); + userMapping.put("properties", userPropertiesMapping); userMapping.put("properties", userPropertiesMapping); propertiesObjectMapping.put("user", userMapping); ObjectMapping.put("properties", propertiesObjectMapping); @@ -68,7 +73,6 @@ public void testMatchAllDocsQuery() throws IOException { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchAllQuery()); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null); assertEquals("select _id from `table` limit 10 offset 0", sql); } @@ -76,21 +80,40 @@ public void testMatchAllDocsQuery() throws IOException { public void testProximaQuery() throws IOException { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(new KnnQueryBuilder("field", new float[] { 1.0f, 2.0f }, 20)); - - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, indexMapping); assertEquals( "select _id, (1/(1+vector_score('field'))) as _score from `table` where " + "MATCHINDEX('field', '1.0,2.0&n=20') order by _score desc limit 10 offset 0", sql ); + + SearchSourceBuilder objectSearcherBuilder = new SearchSourceBuilder(); + objectSearcherBuilder.query(new KnnQueryBuilder("user_image_vector", new float[] { 1.0f, 2.0f }, 20)); + String objectSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", objectSearcherBuilder, ObjectMapping); + assertEquals( + "select _id, (1/(1+vector_score('user_image_vector'))) as _score from `table` where " + + "MATCHINDEX('user_image_vector', '1.0,2.0&n=20') order by _score desc limit 10 offset 0", + objectSql + ); + + SearchSourceBuilder objectSearcherWithDotBuilder = new SearchSourceBuilder(); + objectSearcherWithDotBuilder.query(new KnnQueryBuilder("user.image_vector", new float[] { 1.0f, 2.0f }, 20)); + String objectWithDotSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql( + "table", + objectSearcherWithDotBuilder, + ObjectMapping + ); + assertEquals( + "select _id, (1/(1+vector_score('user_image_vector'))) as _score from `table` where " + + "MATCHINDEX('user_image_vector', '1.0,2.0&n=20') order by _score desc limit 10 offset 0", + objectWithDotSql + ); } public void testUnsupportedDSL() { try { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.existsQuery("field")); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null); fail(); } catch (IOException e) { @@ -101,9 +124,22 @@ public void testUnsupportedDSL() { public void testMatchQuery() throws IOException { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchQuery("field", "value")); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null); assertEquals("select _id from `table` where MATCHINDEX('field', 'value') limit 10 offset 0", sql); + + SearchSourceBuilder objectSearcherBuilder = new SearchSourceBuilder(); + objectSearcherBuilder.query(QueryBuilders.matchQuery("user_first_name", "alice")); + String objectSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", objectSearcherBuilder, ObjectMapping); + assertEquals("select _id from `table` where MATCHINDEX('user_first_name', 'alice') limit 10 offset 0", objectSql); + + SearchSourceBuilder objectSearcherWithDotBuilder = new SearchSourceBuilder(); + objectSearcherWithDotBuilder.query(QueryBuilders.matchQuery("user.first_name", "bob")); + String objectWithDotSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql( + "table", + objectSearcherWithDotBuilder, + ObjectMapping + ); + assertEquals("select _id from `table` where MATCHINDEX('user_first_name', 'bob') limit 10 offset 0", objectWithDotSql); } public void testLimit() throws IOException { @@ -111,7 +147,7 @@ public void testLimit() throws IOException { builder.query(QueryBuilders.matchAllQuery()); builder.from(10); builder.size(10); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); + String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null); assertEquals("select _id from `table` limit 10 offset 10", sql); } @@ -120,7 +156,7 @@ public void testNoFrom() throws IOException { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchAllQuery()); builder.size(10); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); + String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null); assertEquals(sql, "select _id from `table` limit 10 offset 0"); } @@ -129,7 +165,7 @@ public void testNoSize() throws IOException { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchAllQuery()); builder.from(10); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); + String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null); assertEquals(sql, "select _id from `table` limit 10 offset 10"); } @@ -139,7 +175,7 @@ public void testKnnDsl() throws IOException { SearchSourceBuilder l2NormBuilder = new SearchSourceBuilder(); l2NormBuilder.query(QueryBuilders.matchAllQuery()); l2NormBuilder.knnSearch(List.of(new KnnSearchBuilder("field1", new float[] { 1.0f, 2.0f }, 20, 20, null))); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); + String l2NormSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", l2NormBuilder, indexMapping); assertEquals( "select _id, ((1/(1+vector_score('field1')))) as _score from `table` " @@ -162,7 +198,7 @@ public void testObjectKnnDsl() throws IOException { SearchSourceBuilder l2NormBuilder = new SearchSourceBuilder(); l2NormBuilder.query(QueryBuilders.matchAllQuery()); l2NormBuilder.knnSearch(List.of(new KnnSearchBuilder("user.image_vector", new float[] { 1.0f, 2.0f }, 20, 20, null))); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); + String l2NormSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", l2NormBuilder, ObjectMapping); assertEquals( "select _id, ((1/(1+vector_score('user_image_vector')))) as _score from `table` " @@ -181,7 +217,7 @@ public void testMultiKnnDsl() throws IOException { new KnnSearchBuilder("field2", new float[] { 0.6f, 0.8f }, 10, 10, null) ) ); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); + String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, indexMapping); assertEquals( "select _id, ((1/(1+vector_score('field1'))) + ((1+vector_score('field2'))/2)) as _score from `table` " @@ -196,7 +232,6 @@ public void testIllegalKnnParams() throws IOException { dotProductBuilder.query(QueryBuilders.matchAllQuery()); dotProductBuilder.knnSearch(List.of(new KnnSearchBuilder("field2", new float[] { 1.0f, 2.0f }, 20, 20, null))); try { - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", dotProductBuilder, indexMapping); fail("should throw IllegalArgumentException"); } catch (IllegalArgumentException e) { @@ -210,7 +245,7 @@ public void testUnsupportedKnnDsl() { SearchSourceBuilder builder = new SearchSourceBuilder(); builder.query(QueryBuilders.matchAllQuery()); builder.knnSearch(List.of(new KnnSearchBuilder("field", new float[] { 1.0f, 2.0f }, 20, 20, 1.0f))); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); + havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, indexMapping); fail(); } catch (IOException e) { @@ -229,7 +264,7 @@ public void testUnsupportedKnnDsl() { KnnSearchBuilder knnSearchBuilder = new KnnSearchBuilder("field", new float[] { 1.0f, 2.0f }, 20, 20, null); knnSearchBuilder.addFilterQuery(QueryBuilders.matchAllQuery()); builder.knnSearch(List.of(knnSearchBuilder)); - HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient); + havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, indexMapping); fail(); } catch (IOException e) {