Skip to content

Commit

Permalink
First round of testing
Browse files Browse the repository at this point in the history
  • Loading branch information
carlosdelest committed Sep 6, 2024
1 parent 05fc57a commit a3e4852
Show file tree
Hide file tree
Showing 9 changed files with 219 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ date now()
"double percentile(number:double|integer|long, percentile:double|integer|long)"
double pi()
"double pow(base:double|integer|long|unsigned_long, exponent:double|integer|long|unsigned_long)"
"boolean qstr(query:keyword|text)"
"keyword repeat(string:keyword|text, number:integer)"
"keyword replace(string:keyword|text, regex:keyword|text, newString:keyword|text)"
"keyword right(string:keyword|text, length:integer)"
Expand Down Expand Up @@ -188,6 +189,7 @@ now |null |null
percentile |[number, percentile] |["double|integer|long", "double|integer|long"] |[, ]
pi |null |null |null
pow |[base, exponent] |["double|integer|long|unsigned_long", "double|integer|long|unsigned_long"] |["Numeric expression for the base. If `null`\, the function returns `null`.", "Numeric expression for the exponent. If `null`\, the function returns `null`."]
qstr |query |["keyword|text"] |["Query string in Lucene query string format."]
repeat |[string, number] |["keyword|text", integer] |[String expression., Number times to repeat.]
replace |[string, regex, newString] |["keyword|text", "keyword|text", "keyword|text"] |[String expression., Regular expression., Replacement string.]
right |[string, length] |["keyword|text", integer] |[The string from which to returns a substring., The number of characters to return.]
Expand Down Expand Up @@ -312,6 +314,7 @@ now |Returns current date and time.
percentile |Returns the value at which a certain percentage of observed values occur. For example, the 95th percentile is the value which is greater than 95% of the observed values and the 50th percentile is the `MEDIAN`.
pi |Returns {wikipedia}/Pi[Pi], the ratio of a circle's circumference to its diameter.
pow |Returns the value of `base` raised to the power of `exponent`.
qstr |Performs a query string query. Returns true if the provided query string matches the row.
repeat |Returns a string constructed by concatenating `string` with itself the specified `number` of times.
replace |The function substitutes in the string `str` any match of the regular expression `regex` with the replacement string `newStr`.
right |Return the substring that extracts 'length' chars from 'str' starting from the right.
Expand Down Expand Up @@ -438,6 +441,7 @@ now |date
percentile |double |[false, false] |false |true
pi |double |null |false |false
pow |double |[false, false] |false |false
qstr |boolean |false |false |false
repeat |keyword |[false, false] |false |false
replace |keyword |[false, false, false] |false |false
right |keyword |[false, false] |false |false
Expand Down Expand Up @@ -508,5 +512,5 @@ countFunctions#[skip:-8.15.99]
meta functions | stats a = count(*), b = count(*), c = count(*) | mv_expand c;

a:long | b:long | c:long
115 | 115 | 115
116 | 116 | 116
;
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
###############################################
# Tests for QSTR function
#

qstrWithField
required_capability: qstr_function
// tag::qstr-with-field[]
from books | where qstr("author: Faulkner") | keep book_no, author | sort book_no | LIMIT 5;
// end::qstr-with-field[]

book_no:keyword | author:text
2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott]
2713 | William Faulkner
2847 | Colleen Faulkner
2883 | William Faulkner
3293 | Danny Faulkner
;
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.esql.plugin;

import org.elasticsearch.Build;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
import org.elasticsearch.xpack.esql.action.ColumnInfoImpl;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
import org.elasticsearch.xpack.esql.core.type.DataType;
import org.junit.Before;

import java.util.List;

import static org.elasticsearch.test.ListMatcher.matchesList;
import static org.elasticsearch.test.MapMatcher.assertMap;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList;
import static org.hamcrest.Matchers.equalTo;

public class QueryStringFunctionIT extends AbstractEsqlIntegTestCase {

@Before
public void setupIndex() {
createAndPopulateIndex();
}

@Override
protected EsqlQueryResponse run(EsqlQueryRequest request) {
assumeTrue("qstr function available in snapshot builds only", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
return super.run(request);
}

public void testSimpleQueryString() {
var query = """
FROM test
| WHERE qstr("content: dog")
| KEEP id
| SORT id
""";

try (var resp = run(query)) {
assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id")));
assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER")));
// values
List<List<Object>> values = getValuesList(resp);
assertMap(values, matchesList().item(List.of(1)).item(List.of(3)).item(List.of(4)).item(List.of(5)));
}
}

public void testMultiFieldQueryString() {
var query = """
FROM test
| WHERE qstr("dog OR canine")
""";

try (var resp = run(query)) {
assertThat(resp.columns().stream().map(ColumnInfoImpl::name).toList(), equalTo(List.of("id")));
assertThat(resp.columns().stream().map(ColumnInfoImpl::type).map(DataType::toString).toList(), equalTo(List.of("INTEGER")));
// values
List<List<Object>> values = getValuesList(resp);
assertThat(values.size(), equalTo(5));
}
}

private void createAndPopulateIndex() {
var indexName = "test";
var client = client().admin().indices();
var CreateRequest = client.prepareCreate(indexName)
.setSettings(Settings.builder().put("index.number_of_shards", 1))
.setMapping("id", "type=integer", "content", "type=text");
assertAcked(CreateRequest);
client().prepareBulk()
.add(
new IndexRequest(indexName).id("1")
.source("id", 1, "content", "The quick brown animal swiftly jumps over a lazy dog", "title", "A Swift Fox's Journey")
)
.add(
new IndexRequest(indexName).id("2")
.source("id", 2, "content", "A speedy brown fox hops effortlessly over a sluggish canine", "title", "The Fox's Leap")
)
.add(
new IndexRequest(indexName).id("3")
.source("id", 4, "content", "Quick and nimble, the fox vaults over the lazy dog", "title", "Brown Fox in Action")
)
.add(
new IndexRequest(indexName).id("4")
.source(
"id",
5,
"content",
"A fox that is quick and brown jumps over a dog that is quite lazy",
"title",
"Speedy Animals"
)
)
.add(
new IndexRequest(indexName).id("5")
.source(
"id",
6,
"content",
"With agility, a quick brown fox bounds over a slow-moving dog",
"title",
"Foxes and Canines"
)
)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
.get();
ensureYellow(indexName);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.function.Function;
import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
Expand All @@ -27,7 +28,9 @@
public abstract class FullTextFunction extends Function implements EvaluatorMapper {
public static List<NamedWriteableRegistry.Entry> getNamedWriteables() {
List<NamedWriteableRegistry.Entry> entries = new ArrayList<>();
entries.add(QueryStringFunction.ENTRY);
if (EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled()) {
entries.add(QueryStringFunction.ENTRY);
}
return entries;
}

Expand Down Expand Up @@ -60,6 +63,9 @@ public void writeTo(StreamOutput out) throws IOException {
public abstract Query asQuery();

protected static String unquoteQueryString(String quotedString) {
if (quotedString.length() < 2) {
return quotedString;
}
return quotedString.substring(1, quotedString.length() - 1);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,18 @@

package org.elasticsearch.xpack.esql.expression.function.fulltext;

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.compute.operator.EvalOperator;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.expression.Foldables;
import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.expression.function.Example;
import org.elasticsearch.xpack.esql.expression.function.FunctionInfo;
import org.elasticsearch.xpack.esql.expression.function.Param;

import java.io.IOException;
Expand All @@ -30,10 +34,15 @@ public class QueryStringFunction extends FullTextFunction {
QueryStringFunction::new
);

@FunctionInfo(
returnType = "boolean",
description = "Performs a query string query. Returns true if the provided query string matches the row.",
examples = { @Example(file = "qstr-function", tag = "qstr-with-field") }
)
public QueryStringFunction(
Source source,
@Param(
name = "string",
name = "query",
type = { "keyword", "text" },
description = "Query string in Lucene query string format."
) Expression queryString
Expand All @@ -47,7 +56,8 @@ private QueryStringFunction(StreamInput in) throws IOException {

@Override
public Query asQuery() {
return new QueryStringQuery(source(), unquoteQueryString(query().sourceText()), Map.of(),null);
String queryAsString = ((BytesRef)Foldables.valueOf(query())).utf8ToString();
return new QueryStringQuery(source(), queryAsString, Map.of(),null);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,10 @@ public final void test() throws Throwable {
"can't use match command in csv tests",
testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.MATCH_COMMAND.capabilityName())
);
assumeFalse(
"can't use QSTR function in csv tests",
testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.QSTR_FUNCTION.capabilityName())
);

if (Build.current().isSnapshot()) {
assertThat(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.elasticsearch.xpack.esql.core.expression.NamedExpression;
import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
import org.elasticsearch.xpack.esql.io.stream.PlanNameRegistry;
import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput;
Expand Down Expand Up @@ -127,6 +128,7 @@ public static NamedWriteableRegistry writableRegistry() {
entries.addAll(AggregateFunction.getNamedWriteables());
entries.addAll(Block.getNamedWriteables());
entries.addAll(LogicalPlan.getNamedWriteables());
entries.addAll(FullTextFunction.getNamedWriteables());
return new NamedWriteableRegistry(entries);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.elasticsearch.xpack.esql.expression.function.ReferenceAttributeTests;
import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute;
import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction;
import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction;
import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction;
import org.elasticsearch.xpack.esql.plan.AbstractNodeSerializationTests;

Expand All @@ -33,6 +34,7 @@ protected final NamedWriteableRegistry getNamedWriteableRegistry() {
entries.addAll(Attribute.getNamedWriteables());
entries.addAll(EsqlScalarFunction.getNamedWriteables());
entries.addAll(AggregateFunction.getNamedWriteables());
entries.addAll(FullTextFunction.getNamedWriteables());
entries.add(UnsupportedAttribute.ENTRY);
entries.add(UnsupportedAttribute.NAMED_EXPRESSION_ENTRY);
entries.add(UnsupportedAttribute.EXPRESSION_ENTRY);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,55 @@ public void testMultiCountAllWithFilter() {
assertThat(plan.anyMatch(EsQueryExec.class::isInstance), is(true));
}

/**
* Expecting
* LimitExec[1000[INTEGER]]
* \_ExchangeExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na
* me{f}#6, long_noidx{f}#11, salary{f}#7],false]
* \_ProjectExec[[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gender{f}#4, job{f}#9, job.raw{f}#10, languages{f}#5, last_na
* me{f}#6, long_noidx{f}#11, salary{f}#7]]
* \_FieldExtractExec[_meta_field{f}#8, emp_no{f}#2, first_name{f}#3, gen]
* \_EsQueryExec[test], indexMode[standard], query[{"query_string":{"query":"last_name: Smith","fields":[]}}]
*/
public void testQueryStringFunction() {
assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
var plan = plannerOptimizer.plan("""
from test
| where qstr("last_name: Smith")
""", IS_SV_STATS);

var limit = as(plan, LimitExec.class);
var exchange = as(limit.child(), ExchangeExec.class);
var project = as(exchange.child(), ProjectExec.class);
var field = as(project.child(), FieldExtractExec.class);
var query = as(field.child(), EsQueryExec.class);
assertThat(query.limit().fold(), is(1000));
var expected = QueryBuilders.queryStringQuery("last_name: Smith");
assertThat(query.query().toString(), is(expected.toString()));
}

public void testQueryStringFunctionMultipleWhereOperands() {
assumeTrue("skipping because QSTR_FUNCTION is not enabled", EsqlCapabilities.Cap.QSTR_FUNCTION.isEnabled());
String queryText = """
from test
| where qstr("last_name: Smith") and emp_no > 10010
""";
var plan = plannerOptimizer.plan(queryText, IS_SV_STATS);

var limit = as(plan, LimitExec.class);
var exchange = as(limit.child(), ExchangeExec.class);
var project = as(exchange.child(), ProjectExec.class);
var field = as(project.child(), FieldExtractExec.class);
var query = as(field.child(), EsQueryExec.class);
assertThat(query.limit().fold(), is(1000));

Source filterSource = new Source(2, 37, "emp_no > 10000");
var range = wrapWithSingleQuery(queryText, QueryBuilders.rangeQuery("emp_no").gt(10010), "emp_no", filterSource);
var queryString = QueryBuilders.queryStringQuery("last_name: Smith");
var expected = QueryBuilders.boolQuery().must(queryString).must(range);
assertThat(query.query().toString(), is(expected.toString()));
}

/**
* Expecting
* LimitExec[1000[INTEGER]]
Expand Down

0 comments on commit a3e4852

Please sign in to comment.