Extend query size limit using scroll

core/src/main/java/org/opensearch/sql/analysis/Analyzer.java

@@ -126,7 +126,7 @@ public LogicalPlan visitRelation(Relation node, AnalysisContext context) {
     // can be removed when analyzing qualified name. The value (expr type) here doesn't matter.
     curEnv.define(new Symbol(Namespace.INDEX_NAME, node.getTableNameOrAlias()), STRUCT);
 
-    return new LogicalRelation(node.getTableName());
+    return new LogicalRelation(node.getTableName(), table.getMaxResultWindow());
   }
 
   @Override

core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java

@@ -39,7 +39,7 @@ public static LogicalPlan filter(LogicalPlan input, Expression expression) {
   }
 
   public static LogicalPlan relation(String tableName) {
-    return new LogicalRelation(tableName);
+    return new LogicalRelation(tableName, 10000);
   }
 
   public static LogicalPlan rename(

core/src/main/java/org/opensearch/sql/planner/logical/LogicalRelation.java

@@ -20,12 +20,16 @@ public class LogicalRelation extends LogicalPlan {
   @Getter
   private final String relationName;
 
+  @Getter
+  private final Integer maxResultWindow;
+
   /**
    * Constructor of LogicalRelation.
    */
-  public LogicalRelation(String relationName) {
+  public LogicalRelation(String relationName, Integer maxResultWindow) {
     super(ImmutableList.of());
     this.relationName = relationName;
+    this.maxResultWindow = maxResultWindow;
   }
 
   @Override

core/src/main/java/org/opensearch/sql/storage/Table.java

@@ -21,6 +21,13 @@ public interface Table {
    */
   Map<String, ExprType> getFieldTypes();
 
+  /**
+   * Get the max result window setting of the table.
+   */
+  default Integer getMaxResultWindow() {
+    return 10000;
+  }
+
   /**
    * Implement a {@link LogicalPlan} by {@link PhysicalPlan} in storage engine.
    *

core/src/test/java/org/opensearch/sql/analysis/WindowExpressionAnalyzerTest.java

@@ -45,7 +45,7 @@
 @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class)
 class WindowExpressionAnalyzerTest extends AnalyzerTestBase {
 
-  private final LogicalPlan child = new LogicalRelation("test");
+  private final LogicalPlan child = new LogicalRelation("test", 10000);
 
   private WindowExpressionAnalyzer analyzer;
 

core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java

@@ -150,7 +150,7 @@ public void visitShouldReturnDefaultPhysicalOperator() {
   @Test
   public void visitRelationShouldThrowException() {
     assertThrows(UnsupportedOperationException.class,
-        () -> new LogicalRelation("test").accept(implementor, null));
+        () -> new LogicalRelation("test", 10000).accept(implementor, null));
   }
 
   @SuppressWarnings({"rawtypes", "unchecked"})

opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java

@@ -30,6 +30,14 @@ public interface OpenSearchClient {
    */
   Map<String, IndexMapping> getIndexMappings(String... indexExpression);
 
+  /**
+   * Fetch index.max_result_window settings according to index expression given.
+   *
+   * @param indexExpression index expression
+   * @return map from index name to its max result window
+   */
+  Map<String, Integer> getIndexMaxResultWindow(String... indexExpression);
+
   /**
    * Perform search query in the search request.
    *

opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java

@@ -24,11 +24,14 @@
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.MappingMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.collect.ImmutableOpenMap;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.IndexSettings;
 import org.opensearch.sql.opensearch.mapping.IndexMapping;
 import org.opensearch.sql.opensearch.request.OpenSearchRequest;
 import org.opensearch.sql.opensearch.response.OpenSearchResponse;
@@ -86,6 +89,29 @@ public Map<String, IndexMapping> getIndexMappings(String... indexExpression) {
     }
   }
 
+  /**
+   * Fetch index.max_result_window settings according to index expression given.
+   *
+   * @param indexExpression index expression
+   * @return map from index name to its max result window
+   */
+  @Override
+  public Map<String, Integer> getIndexMaxResultWindow(String... indexExpression) {
+    ClusterState state = clusterService.state();
+    ImmutableOpenMap<String, IndexMetadata> indicesMetadata = state.metadata().getIndices();
+    String[] concreteIndices = resolveIndexExpression(state, indexExpression);
+
+    ImmutableMap.Builder<String, Integer> result = ImmutableMap.builder();
+    for (String index : concreteIndices) {
+      Settings settings = indicesMetadata.get(index).getSettings();
+      Integer maxResultWindow = settings.getAsInt("index.max_result_window",
+          IndexSettings.MAX_RESULT_WINDOW_SETTING.getDefault(settings));
+      result.put(index, maxResultWindow);
+    }
+
+    return result.build();
+  }
+
   /**
    * TODO: Scroll doesn't work for aggregation. Support aggregation later.
    */

opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java

@@ -11,12 +11,15 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import lombok.RequiredArgsConstructor;
 import org.opensearch.action.admin.cluster.settings.ClusterGetSettingsRequest;
+import org.opensearch.action.admin.indices.settings.get.GetSettingsRequest;
+import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.action.search.ClearScrollRequest;
 import org.opensearch.client.RequestOptions;
 import org.opensearch.client.RestHighLevelClient;
@@ -26,6 +29,7 @@
 import org.opensearch.client.indices.GetMappingsResponse;
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.metadata.AliasMetadata;
+import org.opensearch.common.collect.ImmutableOpenMap;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.sql.opensearch.mapping.IndexMapping;
 import org.opensearch.sql.opensearch.request.OpenSearchRequest;
@@ -54,6 +58,35 @@ public Map<String, IndexMapping> getIndexMappings(String... indexExpression) {
     }
   }
 
+  @Override
+  public Map<String, Integer> getIndexMaxResultWindow(String... indexExpression) {
+    GetSettingsRequest request = new GetSettingsRequest().indices(indexExpression);
+    try {
+      GetSettingsResponse response = client.indices().getSettings(request, RequestOptions.DEFAULT);
+      ImmutableOpenMap<String, Settings> settings = response.getIndexToSettings();
+      ImmutableOpenMap<String, Settings> defaultSettings = response.getIndexToDefaultSettings();
+      Map<String, Integer> result = new HashMap<>();
+
+      defaultSettings.forEach(entry -> {
+        Integer maxResultWindow = entry.value.getAsInt("index.max_result_window", null);
+        if (maxResultWindow != null) {
+          result.put(entry.key, maxResultWindow);
+        }
+      });
+
+      settings.forEach(entry -> {
+        Integer maxResultWindow = entry.value.getAsInt("index.max_result_window", null);
+        if (maxResultWindow != null) {
+          result.put(entry.key, maxResultWindow);
+        }
+      });
+
+      return result;
+    } catch (IOException e) {
+      throw new IllegalStateException("Failed to get max result window for " + indexExpression, e);
+    }
+  }
+
   @Override
   public OpenSearchResponse search(OpenSearchRequest request) {
     return request.search(

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/OpenSearchLogicalIndexAgg.java

@@ -31,6 +31,11 @@ public class OpenSearchLogicalIndexAgg extends LogicalPlan {
 
   private final String relationName;
 
+  /**
+   * Max Result Window.
+   */
+  private final Integer maxResultWindow;
+
   /**
    * Filter Condition.
    */
@@ -61,12 +66,13 @@ public class OpenSearchLogicalIndexAgg extends LogicalPlan {
   @Builder
   public OpenSearchLogicalIndexAgg(
       String relationName,
-      Expression filter,
+      Integer maxResultWindow, Expression filter,
       List<NamedAggregator> aggregatorList,
       List<NamedExpression> groupByList,
       List<Pair<Sort.SortOption, Expression>> sortList) {
     super(ImmutableList.of());
     this.relationName = relationName;
+    this.maxResultWindow = maxResultWindow;
     this.filter = filter;
     this.aggregatorList = aggregatorList;
     this.groupByList = groupByList;

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/OpenSearchLogicalIndexScan.java

@@ -34,6 +34,11 @@ public class OpenSearchLogicalIndexScan extends LogicalPlan {
    */
   private final String relationName;
 
+  /**
+   * Max Result Window.
+   */
+  private final Integer maxResultWindow;
+
   /**
    * Filter Condition.
    */
@@ -64,12 +69,13 @@ public class OpenSearchLogicalIndexScan extends LogicalPlan {
   @Builder
   public OpenSearchLogicalIndexScan(
       String relationName,
-      Expression filter,
+      Integer maxResultWindow, Expression filter,
       Set<ReferenceExpression> projectList,
       List<Pair<Sort.SortOption, Expression>> sortList,
       Integer limit, Integer offset) {
     super(ImmutableList.of());
     this.relationName = relationName;
+    this.maxResultWindow = maxResultWindow;
     this.filter = filter;
     this.projectList = projectList;
     this.sortList = sortList;

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/MergeAggAndIndexScan.java

@@ -49,6 +49,7 @@ public LogicalPlan apply(LogicalAggregation aggregation,
     return OpenSearchLogicalIndexAgg
         .builder()
         .relationName(indexScan.getRelationName())
+        .maxResultWindow(indexScan.getMaxResultWindow())
         .filter(indexScan.getFilter())
         .aggregatorList(aggregation.getAggregatorList())
         .groupByList(aggregation.getGroupByList())

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/MergeAggAndRelation.java

@@ -47,6 +47,7 @@ public LogicalPlan apply(LogicalAggregation aggregation,
     return OpenSearchLogicalIndexAgg
         .builder()
         .relationName(relation.getRelationName())
+        .maxResultWindow(relation.getMaxResultWindow())
         .aggregatorList(aggregation.getAggregatorList())
         .groupByList(aggregation.getGroupByList())
         .build();

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/MergeFilterAndRelation.java

@@ -47,6 +47,7 @@ public LogicalPlan apply(LogicalFilter filter,
     return OpenSearchLogicalIndexScan
         .builder()
         .relationName(relation.getRelationName())
+        .maxResultWindow(relation.getMaxResultWindow())
         .filter(filter.getCondition())
         .build();
   }

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/MergeLimitAndIndexScan.java

@@ -43,6 +43,7 @@ public LogicalPlan apply(LogicalLimit plan, Captures captures) {
     OpenSearchLogicalIndexScan.OpenSearchLogicalIndexScanBuilder builder =
         OpenSearchLogicalIndexScan.builder();
     builder.relationName(indexScan.getRelationName())
+        .maxResultWindow(indexScan.getMaxResultWindow())
         .filter(indexScan.getFilter())
         .offset(plan.getOffset())
         .limit(plan.getLimit());

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/MergeLimitAndRelation.java

@@ -42,6 +42,7 @@ public LogicalPlan apply(LogicalLimit plan, Captures captures) {
     LogicalRelation relation = captures.get(relationCapture);
     return OpenSearchLogicalIndexScan.builder()
         .relationName(relation.getRelationName())
+        .maxResultWindow(relation.getMaxResultWindow())
         .offset(plan.getOffset())
         .limit(plan.getLimit())
         .build();

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/MergeSortAndIndexAgg.java

@@ -62,6 +62,7 @@ public LogicalPlan apply(LogicalSort sort,
     OpenSearchLogicalIndexAgg indexAgg = captures.get(indexAggCapture);
     return OpenSearchLogicalIndexAgg.builder()
         .relationName(indexAgg.getRelationName())
+        .maxResultWindow(indexAgg.getMaxResultWindow())
         .filter(indexAgg.getFilter())
         .groupByList(indexAgg.getGroupByList())
         .aggregatorList(indexAgg.getAggregatorList())

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/MergeSortAndIndexScan.java

@@ -54,6 +54,7 @@ public LogicalPlan apply(LogicalSort sort,
     return OpenSearchLogicalIndexScan
         .builder()
         .relationName(indexScan.getRelationName())
+        .maxResultWindow(indexScan.getMaxResultWindow())
         .filter(indexScan.getFilter())
         .sortList(mergeSortList(indexScan.getSortList(), sort.getSortList()))
         .build();

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/MergeSortAndRelation.java

@@ -47,6 +47,7 @@ public LogicalPlan apply(LogicalSort sort,
     return OpenSearchLogicalIndexScan
         .builder()
         .relationName(relation.getRelationName())
+        .maxResultWindow(relation.getMaxResultWindow())
         .sortList(sort.getSortList())
         .build();
   }

opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndRelation.java

@@ -58,6 +58,7 @@ public LogicalPlan apply(LogicalProject project,
         OpenSearchLogicalIndexScan
             .builder()
             .relationName(relation.getRelationName())
+            .maxResultWindow(relation.getMaxResultWindow())
             .projectList(findReferenceExpressions(project.getProjectList()))
             .build(),
         project.getProjectList(),

opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java

@@ -81,6 +81,16 @@ public OpenSearchQueryRequest(IndexName indexName, int size,
     this.exprValueFactory = factory;
   }
 
+  /**
+   * Constructor of ElasticsearchQueryRequest.
+   */
+  public OpenSearchQueryRequest(IndexName indexName, SearchSourceBuilder sourceBuilder,
+                                OpenSearchExprValueFactory factory) {
+    this.indexName = indexName;
+    this.sourceBuilder = sourceBuilder;
+    this.exprValueFactory = factory;
+  }
+
   @Override
   public OpenSearchResponse search(Function<SearchRequest, SearchResponse> searchAction,
                                    Function<SearchScrollRequest, SearchResponse> scrollAction) {