[ES|QL] Create Range in PushFiltersToSource for qualified pushable filters on the same field

docs/changelog/111437.yaml

@@ -0,0 +1,5 @@
+pr: 111437
+summary: "[ES|QL] Create `Range` in `PushFiltersToSource` for qualified pushable filters on the same field"
+area: ES|QL
+type: enhancement
+issues: []

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/planner/ExpressionTranslators.java

@@ -7,12 +7,10 @@
 
 package org.elasticsearch.xpack.esql.core.planner;
 
-import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute;
-import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MultiMatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.StringQueryPredicate;
@@ -32,31 +30,17 @@
 import org.elasticsearch.xpack.esql.core.querydsl.query.NotQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.Query;
 import org.elasticsearch.xpack.esql.core.querydsl.query.QueryStringQuery;
-import org.elasticsearch.xpack.esql.core.querydsl.query.RangeQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.RegexQuery;
 import org.elasticsearch.xpack.esql.core.querydsl.query.WildcardQuery;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.util.Check;
 import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 
-import java.time.OffsetTime;
-import java.time.ZonedDateTime;
-import java.time.temporal.TemporalAccessor;
 import java.util.Arrays;
 import java.util.List;
 
 public final class ExpressionTranslators {
 
-    public static final String DATE_FORMAT = "strict_date_optional_time_nanos";
-    public static final String TIME_FORMAT = "strict_hour_minute_second_fraction";
-
-    public static Object valueOf(Expression e) {
-        if (e.foldable()) {
-            return e.fold();
-        }
-        throw new QlIllegalArgumentException("Cannot determine value for {}", e);
-    }
-
     // TODO: see whether escaping is needed
     @SuppressWarnings("rawtypes")
     public static class Likes extends ExpressionTranslator<RegexMatch> {
@@ -193,55 +177,6 @@ private static Query translate(IsNull isNull, TranslatorHandler handler) {
         }
     }
 
-    public static class Ranges extends ExpressionTranslator<Range> {
-
-        @Override
-        protected Query asQuery(Range r, TranslatorHandler handler) {
-            return doTranslate(r, handler);
-        }
-
-        public static Query doTranslate(Range r, TranslatorHandler handler) {
-            return handler.wrapFunctionQuery(r, r.value(), () -> translate(r, handler));
-        }
-
-        private static RangeQuery translate(Range r, TranslatorHandler handler) {
-            Object lower = valueOf(r.lower());
-            Object upper = valueOf(r.upper());
-            String format = null;
-
-            // for a date constant comparison, we need to use a format for the date, to make sure that the format is the same
-            // no matter the timezone provided by the user
-            DateFormatter formatter = null;
-            if (lower instanceof ZonedDateTime || upper instanceof ZonedDateTime) {
-                formatter = DateFormatter.forPattern(DATE_FORMAT);
-            } else if (lower instanceof OffsetTime || upper instanceof OffsetTime) {
-                formatter = DateFormatter.forPattern(TIME_FORMAT);
-            }
-            if (formatter != null) {
-                // RangeQueryBuilder accepts an Object as its parameter, but it will call .toString() on the ZonedDateTime
-                // instance which can have a slightly different format depending on the ZoneId used to create the ZonedDateTime
-                // Since RangeQueryBuilder can handle date as String as well, we'll format it as String and provide the format.
-                if (lower instanceof ZonedDateTime || lower instanceof OffsetTime) {
-                    lower = formatter.format((TemporalAccessor) lower);
-                }
-                if (upper instanceof ZonedDateTime || upper instanceof OffsetTime) {
-                    upper = formatter.format((TemporalAccessor) upper);
-                }
-                format = formatter.pattern();
-            }
-            return new RangeQuery(
-                r.source(),
-                handler.nameOf(r.value()),
-                lower,
-                r.includeLower(),
-                upper,
-                r.includeUpper(),
-                format,
-                r.zoneId()
-            );
-        }
-    }
-
     public static Query or(Source source, Query left, Query right) {
         return boolQuery(source, left, right, false);
     }

x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/RestEsqlTestCase.java

@@ -440,6 +440,38 @@ public void testOutOfRangeComparisons() throws IOException {
         }
     }
 
+    public void testInternalRange() throws IOException {
+        final int NUM_SINGLE_VALUE_ROWS = 100;
+        bulkLoadTestData(NUM_SINGLE_VALUE_ROWS);
+
+        String upperBound = randomFrom(" < ", " <= ");
+        String lowerBound = randomFrom(" > ", " >= ");
+
+        String predicate = "{}" + upperBound + "{} and {}" + lowerBound + "{} and {} != {}";
+
+        List<String> predicates = List.of(
+            format(null, predicate, "integer", NUM_SINGLE_VALUE_ROWS + 1, "integer", -1, "integer", -1),
+            format(null, predicate, "long", NUM_SINGLE_VALUE_ROWS + 1, "long", -1, "long", -1),
+            format(null, predicate, "double", NUM_SINGLE_VALUE_ROWS + 1.0, "double", -1.0, "double", -1),
+            format(null, predicate, "date", "\"9999-12-31\"", "date", "\"1001-01-01\"", "date", "\"1000-01-01\""),
+            format(null, predicate, "keyword", "\"l\"", "keyword", "\"j\"", "keyword", "\"a\""),
+            format(null, predicate, "ip", "\"128.0.0.0\"", "ip", "\"126.0.0.0\"", "ip", "\"129.0.0.0\""),
+            format(null, predicate, "version", "\"2\"", "version", "\"1\"", "version", "\"0\"")
+        );
+
+        for (String p : predicates) {
+            var query = requestObjectBuilder().query(format(null, "from {} | where {}", testIndexName(), p));
+            var result = runEsql(query, List.of(), NO_WARNINGS_REGEX, mode);
+
+            var values = as(result.get("values"), ArrayList.class);
+            assertThat(
+                format(null, "Comparison [{}] should return all rows with single values.", p),
+                values.size(),
+                is(NUM_SINGLE_VALUE_ROWS)
+            );
+        }
+    }
+
     public void testWarningHeadersOnFailedConversions() throws IOException {
         int count = randomFrom(10, 40, 60);
         bulkLoadTestData(count);

x-pack/plugin/esql/qa/testFixtures/src/main/resources/comparison.csv-spec

@@ -116,3 +116,99 @@ from employees
 emp_no:integer
 10001
 ;
+
+rangeInteger
+from employees
+| where emp_no > 10001 and emp_no < 10003
+| keep emp_no, first_name
+;
+
+emp_no:integer |first_name:keyword
+10002          |Bezalel
+;
+
+rangeLong
+from employees
+| where languages.long > 1 and languages.long < 3
+| keep emp_no, first_name
+| sort emp_no
+| limit 2
+;
+
+emp_no:integer |first_name:keyword
+10001          |Georgi
+10008          |Saniya
+;
+
+rangeDouble
+from employees
+| where height > 1.4 and height < 2.0
+| keep emp_no, first_name
+| sort emp_no
+| limit 2
+;
+
+emp_no:integer |first_name:keyword
+10003          |Parto
+10004          |Chirstian
+;
+
+rangeUnsignedLong
+from ul_logs
+| where bytes_out >= to_ul(4747671420480199905) and bytes_out <= to_ul(12749081495402663265)
+| keep id
+| sort id
+| limit 2
+;
+
+id:integer
+1
+3
+;
+
+
+rangeKeyword
+from employees
+| where first_name >= "A" and first_name <= "D"
+| keep emp_no, first_name
+| sort emp_no
+| limit 2
+;
+
+emp_no:integer |first_name:keyword
+10002          |Bezalel
+10004          |Chirstian
+;
+
+rangeVersion
+from apps
+| where version > "2" and version < "4"
+| keep id, version
+| sort id
+;
+
+id:integer |version:version
+2          |2.1
+3          |2.3.4
+4          |2.12.0
+;
+
+rangeDateTime
+from employees
+| where birth_date >= "1952-01-01" and birth_date <= "1952-12-31"
+| stats cnt = count(*)
+;
+
+cnt:long
+8
+;
+
+rangeMixed
+from employees
+| where birth_date >= "1952-01-01" and birth_date <= "1952-12-31" and hire_date >= "1980-01-01" and hire_date <= "1989-12-31"
+| stats cnt = count(*)
+;
+
+cnt:long
+5
+;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/spatial/SpatialRelatesUtils.java

@@ -28,7 +28,7 @@
 
 import java.io.IOException;
 
-import static org.elasticsearch.xpack.esql.core.planner.ExpressionTranslators.valueOf;
+import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
 
 public class SpatialRelatesUtils {
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizer.java

@@ -30,6 +30,7 @@
 import org.elasticsearch.xpack.esql.core.expression.TypedAttribute;
 import org.elasticsearch.xpack.esql.core.expression.function.scalar.UnaryScalarFunction;
 import org.elasticsearch.xpack.esql.core.expression.predicate.Predicates;
+import org.elasticsearch.xpack.esql.core.expression.predicate.Range;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.MatchQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.fulltext.StringQueryPredicate;
 import org.elasticsearch.xpack.esql.core.expression.predicate.logical.And;
@@ -45,6 +46,7 @@
 import org.elasticsearch.xpack.esql.core.rule.Rule;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
+import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
 import org.elasticsearch.xpack.esql.core.util.Queries;
 import org.elasticsearch.xpack.esql.core.util.Queries.Clause;
 import org.elasticsearch.xpack.esql.core.util.StringUtils;
@@ -59,8 +61,12 @@
 import org.elasticsearch.xpack.esql.expression.function.scalar.spatial.StDistance;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.In;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.InsensitiveBinaryComparison;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan;
+import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThanOrEqual;
 import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals;
 import org.elasticsearch.xpack.esql.optimizer.PhysicalOptimizerRules.OptimizerRule;
 import org.elasticsearch.xpack.esql.plan.physical.AggregateExec;
@@ -252,8 +258,10 @@ protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext
                 for (Expression exp : splitAnd(filterExec.condition())) {
                     (canPushToSource(exp, x -> hasIdenticalDelegate(x, ctx.searchStats())) ? pushable : nonPushable).add(exp);
                 }
-                if (pushable.size() > 0) { // update the executable with pushable conditions
-                    Query queryDSL = TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(pushable));
+                // Combine GT, GTE, LT and LTE in pushable to Range if possible
+                List<Expression> newPushable = combineEligiblePushableToRange(pushable);
+                if (newPushable.size() > 0) { // update the executable with pushable conditions
+                    Query queryDSL = TRANSLATOR_HANDLER.asQuery(Predicates.combineAnd(newPushable));
                     QueryBuilder planQuery = queryDSL.asBuilder();
                     var query = Queries.combine(Clause.FILTER, asList(queryExec.query(), planQuery));
                     queryExec = new EsQueryExec(
@@ -277,6 +285,79 @@ protected PhysicalPlan rule(FilterExec filterExec, LocalPhysicalOptimizerContext
             return plan;
         }
 
+        private static List<Expression> combineEligiblePushableToRange(List<Expression> pushable) {
+            List<EsqlBinaryComparison> bcs = new ArrayList<>();
+            List<Range> ranges = new ArrayList<>();
+            List<Expression> others = new ArrayList<>();
+            boolean changed = false;
+
+            pushable.forEach(e -> {
+                if (e instanceof GreaterThan || e instanceof GreaterThanOrEqual || e instanceof LessThan || e instanceof LessThanOrEqual) {
+                    if (((EsqlBinaryComparison) e).right().foldable()) {
+                        bcs.add((EsqlBinaryComparison) e);
+                    } else {
+                        others.add(e);
+                    }
+                } else {
+                    others.add(e);
+                }
+            });
+
+            for (int i = 0, step = 1; i < bcs.size() - 1; i += step, step = 1) {
+                BinaryComparison main = bcs.get(i);
+                for (int j = i + 1; j < bcs.size(); j++) {
+                    BinaryComparison other = bcs.get(j);
+                    if (main.left().semanticEquals(other.left())) {
+                        // >/>= AND </<=
+                        if ((main instanceof GreaterThan || main instanceof GreaterThanOrEqual)
+                            && (other instanceof LessThan || other instanceof LessThanOrEqual)) {
+                            bcs.remove(j);
+                            bcs.remove(i);
+
+                            ranges.add(
+                                new Range(
+                                    main.source(),
+                                    main.left(),
+                                    main.right(),
+                                    main instanceof GreaterThanOrEqual,
+                                    other.right(),
+                                    other instanceof LessThanOrEqual,
+                                    main.zoneId()
+                                )
+                            );
+
+                            changed = true;
+                            step = 0;
+                            break;
+                        }
+                        // </<= AND >/>=
+                        else if ((other instanceof GreaterThan || other instanceof GreaterThanOrEqual)
+                            && (main instanceof LessThan || main instanceof LessThanOrEqual)) {
+                                bcs.remove(j);
+                                bcs.remove(i);
+
+                                ranges.add(
+                                    new Range(
+                                        main.source(),
+                                        main.left(),
+                                        other.right(),
+                                        other instanceof GreaterThanOrEqual,
+                                        main.right(),
+                                        main instanceof LessThanOrEqual,
+                                        main.zoneId()
+                                    )
+                                );
+
+                                changed = true;
+                                step = 0;
+                                break;
+                            }
+                    }
+                }
+            }
+            return changed ? CollectionUtils.combine(others, bcs, ranges) : pushable;
+        }
+
         public static boolean canPushToSource(Expression exp, Predicate<FieldAttribute> hasIdenticalDelegate) {
             if (exp instanceof BinaryComparison bc) {
                 return isAttributePushable(bc.left(), bc, hasIdenticalDelegate) && bc.right().foldable();