From 9fdf4215dd0f941dce74a708b532900b85228210 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Fri, 10 May 2019 14:19:26 +0300 Subject: [PATCH] Docs: Documentation for the upcoming SQL support of frozen indices (#41863) (cherry picked from commit a3cc03eb1503df24c1706a721fcc9af38c3b2873) (cherry picked from commit f42dcf2ffd7bd25f3f91aa6127515f393cd1860f) --- docs/reference/sql/endpoints/jdbc.asciidoc | 4 ++ docs/reference/sql/endpoints/rest.asciidoc | 4 ++ docs/reference/sql/language/index.asciidoc | 2 +- ...dex-patterns.asciidoc => indices.asciidoc} | 36 +++++++++++++++++- .../syntax/commands/show-tables.asciidoc | 10 +++-- .../xpack/sql/qa/jdbc/DataLoader.java | 3 ++ .../qa/src/main/resources/docs/docs.csv-spec | 30 +++++++++++++++ .../logical/command/sys/SysTablesTests.java | 38 +++++++++++++++++-- 8 files changed, 116 insertions(+), 11 deletions(-) rename docs/reference/sql/language/{index-patterns.asciidoc => indices.asciidoc} (65%) diff --git a/docs/reference/sql/endpoints/jdbc.asciidoc b/docs/reference/sql/endpoints/jdbc.asciidoc index 33f130a891896..4e9de8934895a 100644 --- a/docs/reference/sql/endpoints/jdbc.asciidoc +++ b/docs/reference/sql/endpoints/jdbc.asciidoc @@ -136,6 +136,10 @@ Query timeout (in seconds). That is the maximum amount of time waiting for a que `field.multi.value.leniency` (default `true`):: Whether to be lenient and return the first value (without any guarantees of what that will be - typically the first in natural ascending order) for fields with multiple values (true) or throw an exception. +[float] +==== Index +`index.include.frozen` (default `false`):: Whether to include <> in the query execution or not (default). + [float] ==== Additional diff --git a/docs/reference/sql/endpoints/rest.asciidoc b/docs/reference/sql/endpoints/rest.asciidoc index c55b379bfffe6..e44649f3a8767 100644 --- a/docs/reference/sql/endpoints/rest.asciidoc +++ b/docs/reference/sql/endpoints/rest.asciidoc @@ -360,6 +360,10 @@ More information available https://docs.oracle.com/javase/8/docs/api/java/time/Z |false |Throw an exception when encountering multiple values for a field (default) or be lenient and return the first value from the list (without any guarantees of what that will be - typically the first in natural ascending order). +|index_include_frozen +|false +|Whether to include <> in the query execution or not (default). + |=== Do note that most parameters (outside the timeout and `columnar` ones) make sense only during the initial query - any follow-up pagination request only requires the `cursor` parameter as explained in the <> chapter. diff --git a/docs/reference/sql/language/index.asciidoc b/docs/reference/sql/language/index.asciidoc index 6ea6a15b3ed64..2ae33d32f84ee 100644 --- a/docs/reference/sql/language/index.asciidoc +++ b/docs/reference/sql/language/index.asciidoc @@ -13,4 +13,4 @@ This chapter describes the SQL syntax and semantics supported namely: include::syntax/lexic/index.asciidoc[] include::syntax/commands/index.asciidoc[] include::data-types.asciidoc[] -include::index-patterns.asciidoc[] +include::indices.asciidoc[] diff --git a/docs/reference/sql/language/index-patterns.asciidoc b/docs/reference/sql/language/indices.asciidoc similarity index 65% rename from docs/reference/sql/language/index-patterns.asciidoc rename to docs/reference/sql/language/indices.asciidoc index 44f951d36a028..82c7f30fb041e 100644 --- a/docs/reference/sql/language/index-patterns.asciidoc +++ b/docs/reference/sql/language/indices.asciidoc @@ -5,7 +5,9 @@ {es-sql} supports two types of patterns for matching multiple indices or tables: -* {es} multi-index +[[sql-index-patterns-multi]] +[float] +=== {es} multi-index The {es} notation for enumerating, including or excluding <> is supported _as long_ as it is quoted or escaped as a table identifier. @@ -33,7 +35,9 @@ include-tagged::{sql-specs}/docs/docs.csv-spec[fromTablePatternQuoted] NOTE: There is the restriction that all resolved concrete tables have the exact same mapping. -* SQL `LIKE` notation +[[sql-index-patterns-like]] +[float] +=== SQL `LIKE` notation The common `LIKE` statement (including escaping if needed) to match a wildcard pattern, based on one `_` or multiple `%` characters. @@ -81,3 +85,31 @@ Which one to use, is up to you however try to stick to the same one across your NOTE: As the query type of quoting between the two patterns is fairly similar (`"` vs `'`), {es-sql} _always_ requires the keyword `LIKE` for SQL `LIKE` pattern. +[[sql-index-frozen]] +== Frozen Indices + +{es} <> are a useful and powerful tool for hot/warm architecture introduced in {es} 6.6, +essentially by trading speed for memory. +{es-sql} supports frozen indices and similar to {es}, due to their performance characteristics, allows searches on them only +when explicitly told so by user - in other words, by default, frozen indices are not included in searches. + +One can toggle the use of frozen indices through: + +dedicated configuration parameter:: +Set to `true` properties `index_include_frozen` in the <> or `index.include.frozen` in the drivers to include frozen indices. + +dedicated keyword:: +Explicitly perform the inclusion through the dedicated `FROZEN` keyword in the `FROM` clause or `INCLUDE FROZEN` in the `SHOW` commands: + +["source","sql",subs="attributes,callouts,macros"] +---- +include-tagged::{sql-specs}/docs/docs.csv-spec[showTablesIncludeFrozen] +---- + + +["source","sql",subs="attributes,callouts,macros"] +---- +include-tagged::{sql-specs}/docs/docs.csv-spec[fromTableIncludeFrozen] +---- + +Unless enabled, frozen indices are completely ignored; it is as if they do not exist and as such, queries ran against them are likely to fail. \ No newline at end of file diff --git a/docs/reference/sql/language/syntax/commands/show-tables.asciidoc b/docs/reference/sql/language/syntax/commands/show-tables.asciidoc index 691d328aa4bdd..554819e24b178 100644 --- a/docs/reference/sql/language/syntax/commands/show-tables.asciidoc +++ b/docs/reference/sql/language/syntax/commands/show-tables.asciidoc @@ -7,12 +7,14 @@ [source, sql] ---- SHOW TABLES - [table identifier | <1> - [LIKE pattern ]]? <2> + [INCLUDE FROZEN]? <1> + [table identifier | <2> + [LIKE pattern ]]? <3> ---- -<1> single table identifier or double quoted es multi index -<2> SQL LIKE pattern +<1> Whether or not to include frozen indices +<2> single table identifier or double quoted es multi index +<3> SQL LIKE pattern See <> for more information about patterns. diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java index 92b3d0a110534..ff50a33a0afe8 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java @@ -58,6 +58,9 @@ public static void loadDocsDatasetIntoEs(RestClient client) throws Exception { loadEmpDatasetIntoEs(client, "emp", "employees"); loadLibDatasetIntoEs(client, "library"); makeAlias(client, "employees", "emp"); + // frozen index + loadLibDatasetIntoEs(client, "archive"); + freeze(client, "archive"); } public static void createString(String name, XContentBuilder builder) throws Exception { diff --git a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec index b6975d3a6b4da..936c7eef88191 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec @@ -172,6 +172,25 @@ employees |VIEW |ALIAS // end::showTablesEsMultiIndex ; + +// +// include FROZEN +// +showTablesIncludeFrozen +// tag::showTablesIncludeFrozen +SHOW TABLES INCLUDE FROZEN; + + name | type | kind +---------------+---------------+--------------- +archive |BASE TABLE |FROZEN INDEX +emp |BASE TABLE |INDEX +employees |VIEW |ALIAS +library |BASE TABLE |INDEX + +// end::showTablesIncludeFrozen +; + + /////////////////////////////// // // Show Functions @@ -471,6 +490,17 @@ SELECT * FROM "emp" LIMIT 1; // end::fromTableQuoted ; +fromTableIncludeFrozen +// tag::fromTableIncludeFrozen +SELECT * FROM FROZEN archive LIMIT 1; + + author | name | page_count | release_date +-----------------+--------------------+---------------+-------------------- +James S.A. Corey |Leviathan Wakes |561 |2011-06-02T00:00:00Z + +// end::fromTableIncludeFrozen +; + fromTableQuoted // tag::fromTablePatternQuoted SELECT emp_no FROM "e*p" LIMIT 1; diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java index be32e8e81f9b1..d4db97aba09cf 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTablesTests.java @@ -19,13 +19,17 @@ import org.elasticsearch.xpack.sql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.sql.parser.SqlParser; import org.elasticsearch.xpack.sql.plan.logical.command.Command; +import org.elasticsearch.xpack.sql.proto.Mode; +import org.elasticsearch.xpack.sql.proto.Protocol; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; +import org.elasticsearch.xpack.sql.session.Configuration; import org.elasticsearch.xpack.sql.session.SchemaRowSet; import org.elasticsearch.xpack.sql.session.SqlSession; import org.elasticsearch.xpack.sql.stats.Metrics; import org.elasticsearch.xpack.sql.type.DataTypes; import org.elasticsearch.xpack.sql.type.EsField; import org.elasticsearch.xpack.sql.type.TypesTests; +import org.elasticsearch.xpack.sql.util.DateUtils; import java.util.Comparator; import java.util.Iterator; @@ -52,6 +56,9 @@ public class SysTablesTests extends ESTestCase { private final IndexInfo alias = new IndexInfo("alias", IndexType.ALIAS); private final IndexInfo frozen = new IndexInfo("frozen", IndexType.FROZEN_INDEX); + private final Configuration FROZEN_CFG = new Configuration(DateUtils.UTC, Protocol.FETCH_SIZE, Protocol.REQUEST_TIMEOUT, + Protocol.PAGE_TIMEOUT, null, Mode.PLAIN, null, null, null, false, true); + // // catalog enumeration // @@ -150,6 +157,20 @@ public void testSysTablesNoTypes() throws Exception { }, index, alias); } + public void testSysTablesNoTypesAndFrozen() throws Exception { + executeCommand("SYS TABLES", r -> { + assertEquals(3, r.size()); + assertEquals("frozen", r.column(2)); + assertEquals("BASE TABLE", r.column(3)); + assertTrue(r.advanceRow()); + assertEquals("test", r.column(2)); + assertEquals("BASE TABLE", r.column(3)); + assertTrue(r.advanceRow()); + assertEquals("alias", r.column(2)); + assertEquals("VIEW", r.column(3)); + }, FROZEN_CFG, index, alias, frozen); + } + public void testSysTablesWithLegacyTypes() throws Exception { executeCommand("SYS TABLES TYPE 'TABLE', 'ALIAS'", r -> { assertEquals(2, r.size()); @@ -327,7 +348,7 @@ private SqlTypedParamValue param(Object value) { return new SqlTypedParamValue(DataTypes.fromJava(value).typeName, value); } - private Tuple sql(String sql, List params) { + private Tuple sql(String sql, List params, Configuration cfg) { EsIndex test = new EsIndex("test", mapping); Analyzer analyzer = new Analyzer(TestUtils.TEST_CFG, new FunctionRegistry(), IndexResolution.valid(test), new Verifier(new Metrics())); @@ -336,7 +357,7 @@ private Tuple sql(String sql, List para IndexResolver resolver = mock(IndexResolver.class); when(resolver.clusterName()).thenReturn(CLUSTER_NAME); - SqlSession session = new SqlSession(TestUtils.TEST_CFG, null, null, resolver, null, null, null, null, null); + SqlSession session = new SqlSession(cfg, null, null, resolver, null, null, null, null, null); return new Tuple<>(cmd, session); } @@ -344,10 +365,19 @@ private void executeCommand(String sql, Consumer consumer, IndexIn executeCommand(sql, emptyList(), consumer, infos); } - @SuppressWarnings({ "unchecked", "rawtypes" }) + private void executeCommand(String sql, Consumer consumer, Configuration cfg, IndexInfo... infos) throws Exception { + executeCommand(sql, emptyList(), consumer, cfg, infos); + } + private void executeCommand(String sql, List params, Consumer consumer, IndexInfo... infos) throws Exception { - Tuple tuple = sql(sql, params); + executeCommand(sql, params, consumer, TestUtils.TEST_CFG, infos); + } + + @SuppressWarnings({ "unchecked", "rawtypes" }) + private void executeCommand(String sql, List params, Consumer consumer, Configuration cfg, + IndexInfo... infos) throws Exception { + Tuple tuple = sql(sql, params, cfg); IndexResolver resolver = tuple.v2().indexResolver();