SQL: Backport support for Frozen Indices

docs/reference/sql/endpoints/jdbc.asciidoc

@@ -136,6 +136,10 @@ Query timeout (in seconds). That is the maximum amount of time waiting for a que
 `field.multi.value.leniency` (default `true`):: Whether to be lenient and return the first value (without any guarantees of what that
 will be - typically the first in natural ascending order) for fields with multiple values (true) or throw an exception.
 
+[float]
+==== Index
+`index.include.frozen` (default `false`):: Whether to include <<frozen-indices, frozen-indices>> in the query execution or not (default).
+
 [float]
 ==== Additional
 

docs/reference/sql/endpoints/rest.asciidoc

@@ -360,6 +360,10 @@ More information available https://docs.oracle.com/javase/8/docs/api/java/time/Z
 |false
 |Throw an exception when encountering multiple values for a field (default) or be lenient and return the first value from the list (without any guarantees of what that will be - typically the first in natural ascending order).
 
+|index_include_frozen
+|false
+|Whether to include <<frozen-indices, frozen-indices>> in the query execution or not (default).
+
 |===
 
 Do note that most parameters (outside the timeout and `columnar` ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <<sql-pagination, pagination>> chapter.

docs/reference/sql/language/index.asciidoc

@@ -13,4 +13,4 @@ This chapter describes the SQL syntax and semantics supported namely:
 include::syntax/lexic/index.asciidoc[]
 include::syntax/commands/index.asciidoc[]
 include::data-types.asciidoc[]
-include::index-patterns.asciidoc[]
+include::indices.asciidoc[]

docs/reference/sql/language/index-patterns.asciidoc → docs/reference/sql/language/indices.asciidoc

@@ -5,7 +5,9 @@
 
 {es-sql} supports two types of patterns for matching multiple indices or tables:
 
-* {es} multi-index
+[[sql-index-patterns-multi]]
+[float]
+=== {es} multi-index
 
 The {es} notation for enumerating, including or excluding <<multi-index,multi index syntax>>
 is supported _as long_ as it is quoted or escaped as a table identifier.
@@ -33,7 +35,9 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[fromTablePatternQuoted]
 
 NOTE: There is the restriction that all resolved concrete tables have the exact same mapping.
 
-* SQL `LIKE` notation
+[[sql-index-patterns-like]]
+[float]
+=== SQL `LIKE` notation
 
 The common `LIKE` statement (including escaping if needed) to match a wildcard pattern, based on one `_`
 or multiple `%` characters.
@@ -81,3 +85,31 @@ Which one to use, is up to you however try to stick to the same one across your
 NOTE: As the query type of quoting between the two patterns is fairly similar (`"` vs `'`), {es-sql} _always_
 requires the keyword `LIKE` for SQL `LIKE` pattern.
 
+[[sql-index-frozen]]
+== Frozen Indices
+
+{es} <<frozen-indices, frozen indices>> are a useful and powerful tool for hot/warm architecture introduced in {es} 6.6, 
+essentially by trading speed for memory.
+{es-sql} supports frozen indices and similar to {es}, due to their performance characteristics, allows searches on them only
+when explicitly told so by user - in other words, by default, frozen indices are not included in searches.
+
+One can toggle the use of frozen indices through:
+
+dedicated configuration parameter::
+Set to `true` properties `index_include_frozen` in the <<sql-rest>> or `index.include.frozen` in the drivers to include frozen indices.
+
+dedicated keyword::
+Explicitly perform the inclusion through the dedicated `FROZEN` keyword in the `FROM` clause or `INCLUDE FROZEN` in the `SHOW` commands:
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesIncludeFrozen]
+----
+
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs/docs.csv-spec[fromTableIncludeFrozen]
+----
+
+Unless enabled, frozen indices are completely ignored; it is as if they do not exist and as such, queries ran against them are likely to fail.

docs/reference/sql/language/syntax/commands/show-tables.asciidoc

@@ -7,12 +7,14 @@
 [source, sql]
 ----
 SHOW TABLES
-    [table identifier | <1>
-    [LIKE pattern ]]?   <2>
+    [INCLUDE FROZEN]?   <1>
+    [table identifier | <2>
+    [LIKE pattern ]]?   <3>
 ----
 
-<1> single table identifier or double quoted es multi index
-<2> SQL LIKE pattern
+<1> Whether or not to include frozen indices
+<2> single table identifier or double quoted es multi index
+<3> SQL LIKE pattern
 
 See <<sql-index-patterns, index patterns>> for more information about
 patterns.

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java

@@ -57,10 +57,13 @@ public class JdbcConfiguration extends ConnectionConfiguration {
     static final String FIELD_MULTI_VALUE_LENIENCY = "field.multi.value.leniency";
     static final String FIELD_MULTI_VALUE_LENIENCY_DEFAULT = "true";
 
+    static final String INDEX_INCLUDE_FROZEN = "index.include.frozen";
+    static final String INDEX_INCLUDE_FROZEN_DEFAULT = "false";
+
 
     // options that don't change at runtime
     private static final Set<String> OPTION_NAMES = new LinkedHashSet<>(
-            Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, DEBUG, DEBUG_OUTPUT));
+            Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, INDEX_INCLUDE_FROZEN, DEBUG, DEBUG_OUTPUT));
 
     static {
         // trigger version initialization
@@ -77,6 +80,7 @@ public class JdbcConfiguration extends ConnectionConfiguration {
     // mutable ones
     private ZoneId zoneId;
     private boolean fieldMultiValueLeniency;
+    private boolean includeFrozen;
 
     public static JdbcConfiguration create(String u, Properties props, int loginTimeoutSeconds) throws JdbcSQLException {
         URI uri = parseUrl(u);
@@ -159,6 +163,8 @@ private JdbcConfiguration(URI baseURI, String u, Properties props) throws JdbcSQ
                 s -> TimeZone.getTimeZone(s).toZoneId().normalized());
         this.fieldMultiValueLeniency = parseValue(FIELD_MULTI_VALUE_LENIENCY,
                 props.getProperty(FIELD_MULTI_VALUE_LENIENCY, FIELD_MULTI_VALUE_LENIENCY_DEFAULT), Boolean::parseBoolean);
+        this.includeFrozen = parseValue(INDEX_INCLUDE_FROZEN, props.getProperty(INDEX_INCLUDE_FROZEN, INDEX_INCLUDE_FROZEN_DEFAULT),
+                Boolean::parseBoolean);
     }
 
     @Override
@@ -186,6 +192,10 @@ public boolean fieldMultiValueLeniency() {
         return fieldMultiValueLeniency;
     }
 
+    public boolean indexIncludeFrozen() {
+        return includeFrozen;
+    }
+
     public static boolean canAccept(String url) {
         return (StringUtils.hasText(url) && url.trim().startsWith(JdbcConfiguration.URL_PREFIX));
     }

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java

@@ -63,7 +63,8 @@ Cursor query(String sql, List<SqlTypedParamValue> params, RequestMeta meta) thro
                 Boolean.FALSE,
                 null,
                 new RequestInfo(Mode.JDBC),
-                conCfg.fieldMultiValueLeniency());
+                conCfg.fieldMultiValueLeniency(),
+                conCfg.indexIncludeFrozen());
         SqlQueryResponse response = httpClient.query(sqlRequest);
         return new DefaultCursor(this, response.cursor(), toJdbcColumnInfo(response.columns()), response.rows(), meta);
     }

x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSecurityIT.java

@@ -145,7 +145,8 @@ public void expectShowTables(List<String> tables, String user) throws Exception
                 String tablesOutput = cli.command("SHOW TABLES");
                 assertThat(tablesOutput, containsString("name"));
                 assertThat(tablesOutput, containsString("type"));
-                assertEquals("---------------+---------------", cli.readLine());
+                assertThat(tablesOutput, containsString("kind"));
+                assertEquals("---------------+---------------+---------------", cli.readLine());
                 for (String table : tables) {
                     String line = null;
                     /*

x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java

@@ -122,13 +122,15 @@ public void expectShowTables(List<String> tables, String user) throws Exception
             List<Object> columns = new ArrayList<>();
             columns.add(columnInfo(mode, "name", "keyword", JDBCType.VARCHAR, 32766));
             columns.add(columnInfo(mode, "type", "keyword", JDBCType.VARCHAR, 32766));
+            columns.add(columnInfo(mode, "kind", "keyword", JDBCType.VARCHAR, 32766));
             Map<String, Object> expected = new HashMap<>();
             expected.put("columns", columns);
             List<List<String>> rows = new ArrayList<>();
             for (String table : tables) {
                 List<String> fields = new ArrayList<>();
                 fields.add(table);
                 fields.add("BASE TABLE");
+                fields.add("INDEX");
                 rows.add(fields);
             }
             expected.put("rows", rows);

x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java

@@ -17,12 +17,12 @@
 
 public class JdbcCsvSpecIT extends CsvSpecTestCase {
 
-
     @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
     public static List<Object[]> readScriptSpec() throws Exception {
         List<Object[]> list = new ArrayList<>();
         list.addAll(CsvSpecTestCase.readScriptSpec());
-        return readScriptSpec("/single-node-only/command-sys.csv-spec", specParser());
+        list.addAll(readScriptSpec("/single-node-only/command-sys.csv-spec", specParser()));
+        return list;
     }
 
     public JdbcCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {

x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcFrozenCsvSpecIT.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.qa.single_node;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvSpecTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser;
+
+public class JdbcFrozenCsvSpecIT extends CsvSpecTestCase {
+
+    @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
+    public static List<Object[]> readScriptSpec() throws Exception {
+        return readScriptSpec("/slow/frozen.csv-spec", specParser());
+    }
+
+    @Override
+    protected Properties connectionProperties() {
+        Properties props = new Properties(super.connectionProperties());
+        String timeout = String.valueOf(TimeUnit.MINUTES.toMillis(5));
+        props.setProperty("connect.timeout", timeout);
+        props.setProperty("network.timeout", timeout);
+        props.setProperty("query.timeout", timeout);
+        props.setProperty("page.timeout", timeout);
+
+        return props;
+    }
+
+
+    public JdbcFrozenCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
+        super(fileName, groupName, testName, lineNumber, testCase);
+    }
+}

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
 
@@ -42,14 +43,8 @@ public CsvSpecTestCase(String fileName, String groupName, String testName, Integ
     protected final void doTest() throws Throwable {
         // Run the time tests always in UTC
         // TODO: https://github.com/elastic/elasticsearch/issues/40779
-        if ("time".equals(groupName)) {
-            try (Connection csv = csvConnection(testCase); Connection es = esJdbc(connectionProperties())) {
-                executeAndAssert(csv, es);
-            }
-        } else {
-            try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
-                executeAndAssert(csv, es);
-            }
+        try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
+            executeAndAssert(csv, es);
         }
     }
 

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java

@@ -49,12 +49,18 @@ protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception {
         loadLogsDatasetIntoEs(client, "logs", "logs");
         makeAlias(client, "test_alias", "test_emp", "test_emp_copy");
         makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy");
+        // frozen index
+        loadEmpDatasetIntoEs(client, "frozen_emp", "employees");
+        freeze(client, "frozen_emp");
     }
 
     public static void loadDocsDatasetIntoEs(RestClient client) throws Exception {
         loadEmpDatasetIntoEs(client, "emp", "employees");
         loadLibDatasetIntoEs(client, "library");
         makeAlias(client, "employees", "emp");
+        // frozen index
+        loadLibDatasetIntoEs(client, "archive");
+        freeze(client, "archive");
     }
 
     public static void createString(String name, XContentBuilder builder) throws Exception {
@@ -292,6 +298,12 @@ public static void makeAlias(RestClient client, String aliasName, String... indi
         }
     }
 
+    protected static void freeze(RestClient client, String... indices) throws Exception {
+        for (String index : indices) {
+            client.performRequest(new Request("POST", "/" + index + "/_freeze"));
+        }
+    }
+
     private static void csvToLines(String name, CheckedBiConsumer<List<String>, List<String>, Exception> consumeLine) throws Exception {
         String location = "/" + name + ".csv";
         URL dataSet = SqlSpecTestCase.class.getResource(location);

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ShowTablesTestCase.java

@@ -30,7 +30,7 @@ public void testShowTablesWithManyIndices() throws Exception {
             for (int i = 0; i < indices; i++) {
                 String index = String.format(Locale.ROOT, "test%02d", i);
                 index(index, builder -> builder.field("name", "bob"));
-                h2.createStatement().executeUpdate("INSERT INTO mock VALUES ('" + index + "', 'BASE TABLE');");
+                h2.createStatement().executeUpdate("INSERT INTO mock VALUES ('" + index + "', 'BASE TABLE', 'INDEX');");
             }
 
             ResultSet expected = h2.createStatement().executeQuery("SELECT * FROM mock ORDER BY name");

x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec

@@ -82,20 +82,20 @@ salary              |INTEGER        |integer
 showAlias
 SHOW TABLES LIKE 'test\_alias' ESCAPE '\';
 
-name:s               | type:s
+name:s               | type:s  | kind:s
 
-test_alias           | VIEW
+test_alias           | VIEW    | ALIAS
 ;
 
 showPattern
 SHOW TABLES LIKE 'test_%';
 
-name:s               | type:s
+name:s               | type:s      | kind :s
 
-test_alias           | VIEW
-test_alias_emp       | VIEW
-test_emp             | BASE TABLE
-test_emp_copy        | BASE TABLE
+test_alias           | VIEW        | ALIAS
+test_alias_emp       | VIEW        | ALIAS
+test_emp             | BASE TABLE  | INDEX
+test_emp_copy        | BASE TABLE  | INDEX
 ;
 
 groupByOnAlias