Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

优化term、match查询对object对象fieldname的解析 #415

Merged
merged 1 commit into from
Jan 8, 2024
Merged

优化term、match查询对object对象fieldname的解析 #415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 19 additions & 7 deletions ...avenask-engine/src/main/java/org/havenask/engine/search/HavenaskSearchQueryProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public String transferSearchRequest2HavenaskSql(String table, SearchSourceBuilde
checkVectorMagnitude(similarity, proximaQueryBuilder.getVector());

selectParams.append(", ").append(getScoreComputeStr(fieldName, similarity)).append(" as _score");
where.append(" where MATCHINDEX('").append(proximaQueryBuilder.getFieldName()).append("', '");
where.append(" where MATCHINDEX('").append(fieldName).append("', '");
for (int i = 0; i < proximaQueryBuilder.getVector().length; i++) {

where.append(proximaQueryBuilder.getVector()[i]);
Expand All @@ -158,22 +158,34 @@ public String transferSearchRequest2HavenaskSql(String table, SearchSourceBuilde
orderBy.append(" order by _score desc");
} else if (queryBuilder instanceof TermQueryBuilder) {
TermQueryBuilder termQueryBuilder = (TermQueryBuilder) queryBuilder;
where.append(" where ").append(termQueryBuilder.fieldName()).append("='").append(termQueryBuilder.value()).append("'");
where.append(" where ")
.append(Schema.encodeFieldWithDot(termQueryBuilder.fieldName()))
.append("='")
.append(termQueryBuilder.value())
.append("'");
} else if (queryBuilder instanceof MatchQueryBuilder) {
MatchQueryBuilder matchQueryBuilder = (MatchQueryBuilder) queryBuilder;
where.append(" where MATCHINDEX('")
.append(matchQueryBuilder.fieldName())
.append(Schema.encodeFieldWithDot(matchQueryBuilder.fieldName()))
.append("', '")
.append(matchQueryBuilder.value())
.append("')");
} else {
throw new IOException("unsupported DSL: " + dsl);
}
}
sqlQuery.append("select").append(selectParams).append(" from ").append('`').append(table).append('`');
sqlQuery.append(where).append(orderBy);

sqlQuery.append(" limit ").append(size).append(" offset ").append(from);
sqlQuery.append("select")
.append(selectParams)
.append(" from ")
.append('`')
.append(table)
.append('`')
.append(where)
.append(orderBy)
.append(" limit ")
.append(size)
.append(" offset ")
.append(from);

return sqlQuery.toString();
}
Expand Down
63 changes: 49 additions & 14 deletions ...ine/src/test/java/org/havenask/engine/search/fetch/HavenaskSearchQueryProcessorTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

public class HavenaskSearchQueryProcessorTests extends HavenaskTestCase {
private QrsClient qrsClient = mock(QrsClient.class);
private HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);
private Map<String, Object> indexMapping = new HashMap<>();
private Map<String, Object> ObjectMapping = new HashMap<>();

Expand All @@ -56,9 +57,13 @@ public void setup() {
Map<String, Object> userMapping = new HashMap<>();
Map<String, Object> userPropertiesMapping = new HashMap<>();
Map<String, Object> userImageMapping = new HashMap<>();
Map<String, Object> userFirstNameMapping = new HashMap<>();
userImageMapping.put("type", "vector");
userImageMapping.put("similarity", "L2_NORM");
userPropertiesMapping.put("image_vector", userImageMapping);
userFirstNameMapping.put("type", "keyword");
userPropertiesMapping.put("first_name", userFirstNameMapping);
userMapping.put("properties", userPropertiesMapping);
userMapping.put("properties", userPropertiesMapping);
propertiesObjectMapping.put("user", userMapping);
ObjectMapping.put("properties", propertiesObjectMapping);
Expand All @@ -68,29 +73,47 @@ public void testMatchAllDocsQuery() throws IOException {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());

HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);
String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null);
assertEquals("select _id from `table` limit 10 offset 0", sql);
}

public void testProximaQuery() throws IOException {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(new KnnQueryBuilder("field", new float[] { 1.0f, 2.0f }, 20));

HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);
String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, indexMapping);
assertEquals(
"select _id, (1/(1+vector_score('field'))) as _score from `table` where "
+ "MATCHINDEX('field', '1.0,2.0&n=20') order by _score desc limit 10 offset 0",
sql
);

SearchSourceBuilder objectSearcherBuilder = new SearchSourceBuilder();
objectSearcherBuilder.query(new KnnQueryBuilder("user_image_vector", new float[] { 1.0f, 2.0f }, 20));
String objectSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", objectSearcherBuilder, ObjectMapping);
assertEquals(
"select _id, (1/(1+vector_score('user_image_vector'))) as _score from `table` where "
+ "MATCHINDEX('user_image_vector', '1.0,2.0&n=20') order by _score desc limit 10 offset 0",
objectSql
);

SearchSourceBuilder objectSearcherWithDotBuilder = new SearchSourceBuilder();
objectSearcherWithDotBuilder.query(new KnnQueryBuilder("user.image_vector", new float[] { 1.0f, 2.0f }, 20));
String objectWithDotSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql(
"table",
objectSearcherWithDotBuilder,
ObjectMapping
);
assertEquals(
"select _id, (1/(1+vector_score('user_image_vector'))) as _score from `table` where "
+ "MATCHINDEX('user_image_vector', '1.0,2.0&n=20') order by _score desc limit 10 offset 0",
objectWithDotSql
);
}

public void testUnsupportedDSL() {
try {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.existsQuery("field"));
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);
havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null);
fail();
} catch (IOException e) {
Expand All @@ -101,17 +124,30 @@ public void testUnsupportedDSL() {
public void testMatchQuery() throws IOException {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchQuery("field", "value"));
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);
String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null);
assertEquals("select _id from `table` where MATCHINDEX('field', 'value') limit 10 offset 0", sql);

SearchSourceBuilder objectSearcherBuilder = new SearchSourceBuilder();
objectSearcherBuilder.query(QueryBuilders.matchQuery("user_first_name", "alice"));
String objectSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", objectSearcherBuilder, ObjectMapping);
assertEquals("select _id from `table` where MATCHINDEX('user_first_name', 'alice') limit 10 offset 0", objectSql);

SearchSourceBuilder objectSearcherWithDotBuilder = new SearchSourceBuilder();
objectSearcherWithDotBuilder.query(QueryBuilders.matchQuery("user.first_name", "bob"));
String objectWithDotSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql(
"table",
objectSearcherWithDotBuilder,
ObjectMapping
);
assertEquals("select _id from `table` where MATCHINDEX('user_first_name', 'bob') limit 10 offset 0", objectWithDotSql);
}

public void testLimit() throws IOException {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.from(10);
builder.size(10);
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);

String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null);
assertEquals("select _id from `table` limit 10 offset 10", sql);
}
Expand All @@ -120,7 +156,7 @@ public void testNoFrom() throws IOException {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.size(10);
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);

String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null);
assertEquals(sql, "select _id from `table` limit 10 offset 0");
}
Expand All @@ -129,7 +165,7 @@ public void testNoSize() throws IOException {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.from(10);
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);

String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, null);
assertEquals(sql, "select _id from `table` limit 10 offset 10");
}
Expand All @@ -139,7 +175,7 @@ public void testKnnDsl() throws IOException {
SearchSourceBuilder l2NormBuilder = new SearchSourceBuilder();
l2NormBuilder.query(QueryBuilders.matchAllQuery());
l2NormBuilder.knnSearch(List.of(new KnnSearchBuilder("field1", new float[] { 1.0f, 2.0f }, 20, 20, null)));
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);

String l2NormSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", l2NormBuilder, indexMapping);
assertEquals(
"select _id, ((1/(1+vector_score('field1')))) as _score from `table` "
Expand All @@ -162,7 +198,7 @@ public void testObjectKnnDsl() throws IOException {
SearchSourceBuilder l2NormBuilder = new SearchSourceBuilder();
l2NormBuilder.query(QueryBuilders.matchAllQuery());
l2NormBuilder.knnSearch(List.of(new KnnSearchBuilder("user.image_vector", new float[] { 1.0f, 2.0f }, 20, 20, null)));
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);

String l2NormSql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", l2NormBuilder, ObjectMapping);
assertEquals(
"select _id, ((1/(1+vector_score('user_image_vector')))) as _score from `table` "
Expand All @@ -181,7 +217,7 @@ public void testMultiKnnDsl() throws IOException {
new KnnSearchBuilder("field2", new float[] { 0.6f, 0.8f }, 10, 10, null)
)
);
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);

String sql = havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, indexMapping);
assertEquals(
"select _id, ((1/(1+vector_score('field1'))) + ((1+vector_score('field2'))/2)) as _score from `table` "
Expand All @@ -196,7 +232,6 @@ public void testIllegalKnnParams() throws IOException {
dotProductBuilder.query(QueryBuilders.matchAllQuery());
dotProductBuilder.knnSearch(List.of(new KnnSearchBuilder("field2", new float[] { 1.0f, 2.0f }, 20, 20, null)));
try {
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);
havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", dotProductBuilder, indexMapping);
fail("should throw IllegalArgumentException");
} catch (IllegalArgumentException e) {
Expand All @@ -210,7 +245,7 @@ public void testUnsupportedKnnDsl() {
SearchSourceBuilder builder = new SearchSourceBuilder();
builder.query(QueryBuilders.matchAllQuery());
builder.knnSearch(List.of(new KnnSearchBuilder("field", new float[] { 1.0f, 2.0f }, 20, 20, 1.0f)));
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);

havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, indexMapping);
fail();
} catch (IOException e) {
Expand All @@ -229,7 +264,7 @@ public void testUnsupportedKnnDsl() {
KnnSearchBuilder knnSearchBuilder = new KnnSearchBuilder("field", new float[] { 1.0f, 2.0f }, 20, 20, null);
knnSearchBuilder.addFilterQuery(QueryBuilders.matchAllQuery());
builder.knnSearch(List.of(knnSearchBuilder));
HavenaskSearchQueryProcessor havenaskSearchQueryProcessor = new HavenaskSearchQueryProcessor(qrsClient);

havenaskSearchQueryProcessor.transferSearchRequest2HavenaskSql("table", builder, indexMapping);
fail();
} catch (IOException e) {
Expand Down