Docs: Documentation for the upcoming SQL support of frozen indices (e…

…lastic#41863)

(cherry picked from commit a3cc03e)
(cherry picked from commit f42dcf2)

docs/reference/sql/endpoints/jdbc.asciidoc

@@ -136,6 +136,10 @@ Query timeout (in seconds). That is the maximum amount of time waiting for a que
 `field.multi.value.leniency` (default `true`):: Whether to be lenient and return the first value (without any guarantees of what that
 will be - typically the first in natural ascending order) for fields with multiple values (true) or throw an exception.
 
+[float]
+==== Index
+`index.include.frozen` (default `false`):: Whether to include <<frozen-indices, frozen-indices>> in the query execution or not (default).
+
 [float]
 ==== Additional
 

docs/reference/sql/endpoints/rest.asciidoc

@@ -360,6 +360,10 @@ More information available https://docs.oracle.com/javase/8/docs/api/java/time/Z
 |false
 |Throw an exception when encountering multiple values for a field (default) or be lenient and return the first value from the list (without any guarantees of what that will be - typically the first in natural ascending order).
 
+|index_include_frozen
+|false
+|Whether to include <<frozen-indices, frozen-indices>> in the query execution or not (default).
+
 |===
 
 Do note that most parameters (outside the timeout and `columnar` ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <<sql-pagination, pagination>> chapter.

docs/reference/sql/language/index.asciidoc

@@ -13,4 +13,4 @@ This chapter describes the SQL syntax and semantics supported namely:
 include::syntax/lexic/index.asciidoc[]
 include::syntax/commands/index.asciidoc[]
 include::data-types.asciidoc[]
-include::index-patterns.asciidoc[]
+include::indices.asciidoc[]

docs/reference/sql/language/index-patterns.asciidoc → docs/reference/sql/language/indices.asciidoc

@@ -5,7 +5,9 @@
 
 {es-sql} supports two types of patterns for matching multiple indices or tables:
 
-* {es} multi-index
+[[sql-index-patterns-multi]]
+[float]
+=== {es} multi-index
 
 The {es} notation for enumerating, including or excluding <<multi-index,multi index syntax>>
 is supported _as long_ as it is quoted or escaped as a table identifier.
@@ -33,7 +35,9 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[fromTablePatternQuoted]
 
 NOTE: There is the restriction that all resolved concrete tables have the exact same mapping.
 
-* SQL `LIKE` notation
+[[sql-index-patterns-like]]
+[float]
+=== SQL `LIKE` notation
 
 The common `LIKE` statement (including escaping if needed) to match a wildcard pattern, based on one `_`
 or multiple `%` characters.
@@ -81,3 +85,31 @@ Which one to use, is up to you however try to stick to the same one across your
 NOTE: As the query type of quoting between the two patterns is fairly similar (`"` vs `'`), {es-sql} _always_
 requires the keyword `LIKE` for SQL `LIKE` pattern.
 
+[[sql-index-frozen]]
+== Frozen Indices
+
+{es} <<frozen-indices, frozen indices>> are a useful and powerful tool for hot/warm architecture introduced in {es} 6.6, 
+essentially by trading speed for memory.
+{es-sql} supports frozen indices and similar to {es}, due to their performance characteristics, allows searches on them only
+when explicitly told so by user - in other words, by default, frozen indices are not included in searches.
+
+One can toggle the use of frozen indices through:
+
+dedicated configuration parameter::
+Set to `true` properties `index_include_frozen` in the <<sql-rest>> or `index.include.frozen` in the drivers to include frozen indices.
+
+dedicated keyword::
+Explicitly perform the inclusion through the dedicated `FROZEN` keyword in the `FROM` clause or `INCLUDE FROZEN` in the `SHOW` commands:
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesIncludeFrozen]
+----
+
+
+["source","sql",subs="attributes,callouts,macros"]
+----
+include-tagged::{sql-specs}/docs/docs.csv-spec[fromTableIncludeFrozen]
+----
+
+Unless enabled, frozen indices are completely ignored; it is as if they do not exist and as such, queries ran against them are likely to fail.

docs/reference/sql/language/syntax/commands/show-tables.asciidoc

@@ -7,12 +7,14 @@
 [source, sql]
 ----
 SHOW TABLES
-    [table identifier | <1>
-    [LIKE pattern ]]?   <2>
+    [INCLUDE FROZEN]?   <1>
+    [table identifier | <2>
+    [LIKE pattern ]]?   <3>
 ----
 
-<1> single table identifier or double quoted es multi index
-<2> SQL LIKE pattern
+<1> Whether or not to include frozen indices
+<2> single table identifier or double quoted es multi index
+<3> SQL LIKE pattern
 
 See <<sql-index-patterns, index patterns>> for more information about
 patterns.

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java

@@ -58,6 +58,9 @@ public static void loadDocsDatasetIntoEs(RestClient client) throws Exception {
         loadEmpDatasetIntoEs(client, "emp", "employees");
         loadLibDatasetIntoEs(client, "library");
         makeAlias(client, "employees", "emp");
+        // frozen index
+        loadLibDatasetIntoEs(client, "archive");
+        freeze(client, "archive");
     }
 
     public static void createString(String name, XContentBuilder builder) throws Exception {

x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec

@@ -172,6 +172,25 @@ employees      |VIEW           |ALIAS
 // end::showTablesEsMultiIndex
 ;
 
+
+//
+// include FROZEN
+//
+showTablesIncludeFrozen
+// tag::showTablesIncludeFrozen
+SHOW TABLES INCLUDE FROZEN;
+
+     name      |     type      |     kind      
+---------------+---------------+---------------
+archive        |BASE TABLE     |FROZEN INDEX
+emp            |BASE TABLE     |INDEX          
+employees      |VIEW           |ALIAS          
+library        |BASE TABLE     |INDEX    
+
+// end::showTablesIncludeFrozen
+;
+
+
 ///////////////////////////////
 //
 // Show Functions
@@ -471,6 +490,17 @@ SELECT * FROM "emp" LIMIT 1;
 // end::fromTableQuoted
 ;
 
+fromTableIncludeFrozen
+// tag::fromTableIncludeFrozen
+SELECT * FROM FROZEN archive LIMIT 1;
+
+     author      |        name        |  page_count   |    release_date    
+-----------------+--------------------+---------------+--------------------
+James S.A. Corey |Leviathan Wakes     |561            |2011-06-02T00:00:00Z
+
+// end::fromTableIncludeFrozen
+;
+
 fromTableQuoted
 // tag::fromTablePatternQuoted
 SELECT emp_no FROM "e*p" LIMIT 1;

x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java

@@ -19,13 +19,17 @@
 import org.elasticsearch.xpack.sql.expression.function.FunctionRegistry;
 import org.elasticsearch.xpack.sql.parser.SqlParser;
 import org.elasticsearch.xpack.sql.plan.logical.command.Command;
+import org.elasticsearch.xpack.sql.proto.Mode;
+import org.elasticsearch.xpack.sql.proto.Protocol;
 import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue;
+import org.elasticsearch.xpack.sql.session.Configuration;
 import org.elasticsearch.xpack.sql.session.SchemaRowSet;
 import org.elasticsearch.xpack.sql.session.SqlSession;
 import org.elasticsearch.xpack.sql.stats.Metrics;
 import org.elasticsearch.xpack.sql.type.DataTypes;
 import org.elasticsearch.xpack.sql.type.EsField;
 import org.elasticsearch.xpack.sql.type.TypesTests;
+import org.elasticsearch.xpack.sql.util.DateUtils;
 
 import java.util.Comparator;
 import java.util.Iterator;
@@ -52,6 +56,9 @@ public class SysTablesTests extends ESTestCase {
     private final IndexInfo alias = new IndexInfo("alias", IndexType.ALIAS);
     private final IndexInfo frozen = new IndexInfo("frozen", IndexType.FROZEN_INDEX);
 
+    private final Configuration FROZEN_CFG = new Configuration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT,
+            Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null, false, true);
+
     //
     // catalog enumeration
     //
@@ -150,6 +157,20 @@ public void testSysTablesNoTypes() throws Exception {
         }, index, alias);
     }
 
+    public void testSysTablesNoTypesAndFrozen() throws Exception {
+        executeCommand("SYS TABLES", r -> {
+            assertEquals(3, r.size());
+            assertEquals("frozen", r.column(2));
+            assertEquals("BASE TABLE", r.column(3));
+            assertTrue(r.advanceRow());
+            assertEquals("test", r.column(2));
+            assertEquals("BASE TABLE", r.column(3));
+            assertTrue(r.advanceRow());
+            assertEquals("alias", r.column(2));
+            assertEquals("VIEW", r.column(3));
+        }, FROZEN_CFG, index, alias, frozen);
+    }
+
     public void testSysTablesWithLegacyTypes() throws Exception {
         executeCommand("SYS TABLES TYPE 'TABLE', 'ALIAS'", r -> {
             assertEquals(2, r.size());
@@ -327,7 +348,7 @@ private SqlTypedParamValue param(Object value) {
         return new SqlTypedParamValue(DataTypes.fromJava(value).typeName, value);
     }
 
-    private Tuple<Command, SqlSession> sql(String sql, List<SqlTypedParamValue> params) {
+    private Tuple<Command, SqlSession> sql(String sql, List<SqlTypedParamValue> params, Configuration cfg) {
         EsIndex test = new EsIndex("test", mapping);
         Analyzer analyzer = new Analyzer(TestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test),
                                          new Verifier(new Metrics()));
@@ -336,18 +357,27 @@ private Tuple<Command, SqlSession> sql(String sql, List<SqlTypedParamValue> para
         IndexResolver resolver = mock(IndexResolver.class);
         when(resolver.clusterName()).thenReturn(CLUSTER_NAME);
 
-        SqlSession session = new SqlSession(TestUtils.TEST_CFG, null, null, resolver, null, null, null, null, null);
+        SqlSession session = new SqlSession(cfg, null, null, resolver, null, null, null, null, null);
         return new Tuple<>(cmd, session);
     }
 
     private void executeCommand(String sql, Consumer<SchemaRowSet> consumer, IndexInfo... infos) throws Exception {
         executeCommand(sql, emptyList(), consumer, infos);
     }
 
-    @SuppressWarnings({ "unchecked", "rawtypes" })
+    private void executeCommand(String sql, Consumer<SchemaRowSet> consumer, Configuration cfg, IndexInfo... infos) throws Exception {
+        executeCommand(sql, emptyList(), consumer, cfg, infos);
+    }
+
     private void executeCommand(String sql, List<SqlTypedParamValue> params, Consumer<SchemaRowSet> consumer, IndexInfo... infos)
             throws Exception {
-        Tuple<Command, SqlSession> tuple = sql(sql, params);
+        executeCommand(sql, params, consumer, TestUtils.TEST_CFG, infos);
+    }
+
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    private void executeCommand(String sql, List<SqlTypedParamValue> params, Consumer<SchemaRowSet> consumer, Configuration cfg,
+            IndexInfo... infos) throws Exception {
+        Tuple<Command, SqlSession> tuple = sql(sql, params, cfg);
 
         IndexResolver resolver = tuple.v2().indexResolver();