SQL: Add support for FROZEN indices (elastic#41558)

Allow querying of FROZEN indices both through dedicated SQL grammar
extension:
> SELECT field FROM FROZEN index
and also through driver configuration parameter, namely:
> index.include.frozen: true/false

Fix elastic#39390
Fix elastic#39377

(cherry picked from commit 2445a93)

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java

@@ -57,10 +57,13 @@ public class JdbcConfiguration extends ConnectionConfiguration {
     static final String FIELD_MULTI_VALUE_LENIENCY = "field.multi.value.leniency";
     static final String FIELD_MULTI_VALUE_LENIENCY_DEFAULT = "true";
 
+    static final String INDEX_INCLUDE_FROZEN = "index.include.frozen";
+    static final String INDEX_INCLUDE_FROZEN_DEFAULT = "false";
+
 
     // options that don't change at runtime
     private static final Set<String> OPTION_NAMES = new LinkedHashSet<>(
-            Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, DEBUG, DEBUG_OUTPUT));
+            Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, INDEX_INCLUDE_FROZEN, DEBUG, DEBUG_OUTPUT));
 
     static {
         // trigger version initialization
@@ -77,6 +80,7 @@ public class JdbcConfiguration extends ConnectionConfiguration {
     // mutable ones
     private ZoneId zoneId;
     private boolean fieldMultiValueLeniency;
+    private boolean includeFrozen;
 
     public static JdbcConfiguration create(String u, Properties props, int loginTimeoutSeconds) throws JdbcSQLException {
         URI uri = parseUrl(u);
@@ -159,6 +163,8 @@ private JdbcConfiguration(URI baseURI, String u, Properties props) throws JdbcSQ
                 s -> TimeZone.getTimeZone(s).toZoneId().normalized());
         this.fieldMultiValueLeniency = parseValue(FIELD_MULTI_VALUE_LENIENCY,
                 props.getProperty(FIELD_MULTI_VALUE_LENIENCY, FIELD_MULTI_VALUE_LENIENCY_DEFAULT), Boolean::parseBoolean);
+        this.includeFrozen = parseValue(INDEX_INCLUDE_FROZEN, props.getProperty(INDEX_INCLUDE_FROZEN, INDEX_INCLUDE_FROZEN_DEFAULT),
+                Boolean::parseBoolean);
     }
 
     @Override
@@ -186,6 +192,10 @@ public boolean fieldMultiValueLeniency() {
         return fieldMultiValueLeniency;
     }
 
+    public boolean indexIncludeFrozen() {
+        return includeFrozen;
+    }
+
     public static boolean canAccept(String url) {
         return (StringUtils.hasText(url) && url.trim().startsWith(JdbcConfiguration.URL_PREFIX));
     }

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java

@@ -63,7 +63,8 @@ Cursor query(String sql, List<SqlTypedParamValue> params, RequestMeta meta) thro
                 Boolean.FALSE,
                 null,
                 new RequestInfo(Mode.JDBC),
-                conCfg.fieldMultiValueLeniency());
+                conCfg.fieldMultiValueLeniency(),
+                conCfg.indexIncludeFrozen());
         SqlQueryResponse response = httpClient.query(sqlRequest);
         return new DefaultCursor(this, response.cursor(), toJdbcColumnInfo(response.columns()), response.rows(), meta);
     }

x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSecurityIT.java

@@ -145,7 +145,8 @@ public void expectShowTables(List<String> tables, String user) throws Exception
                 String tablesOutput = cli.command("SHOW TABLES");
                 assertThat(tablesOutput, containsString("name"));
                 assertThat(tablesOutput, containsString("type"));
-                assertEquals("---------------+---------------", cli.readLine());
+                assertThat(tablesOutput, containsString("kind"));
+                assertEquals("---------------+---------------+---------------", cli.readLine());
                 for (String table : tables) {
                     String line = null;
                     /*

x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java

@@ -122,13 +122,15 @@ public void expectShowTables(List<String> tables, String user) throws Exception
             List<Object> columns = new ArrayList<>();
             columns.add(columnInfo(mode, "name", "keyword", JDBCType.VARCHAR, 32766));
             columns.add(columnInfo(mode, "type", "keyword", JDBCType.VARCHAR, 32766));
+            columns.add(columnInfo(mode, "kind", "keyword", JDBCType.VARCHAR, 32766));
             Map<String, Object> expected = new HashMap<>();
             expected.put("columns", columns);
             List<List<String>> rows = new ArrayList<>();
             for (String table : tables) {
                 List<String> fields = new ArrayList<>();
                 fields.add(table);
                 fields.add("BASE TABLE");
+                fields.add("INDEX");
                 rows.add(fields);
             }
             expected.put("rows", rows);

x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java

@@ -17,12 +17,12 @@
 
 public class JdbcCsvSpecIT extends CsvSpecTestCase {
 
-
     @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
     public static List<Object[]> readScriptSpec() throws Exception {
         List<Object[]> list = new ArrayList<>();
         list.addAll(CsvSpecTestCase.readScriptSpec());
-        return readScriptSpec("/single-node-only/command-sys.csv-spec", specParser());
+        list.addAll(readScriptSpec("/single-node-only/command-sys.csv-spec", specParser()));
+        return list;
     }
 
     public JdbcCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {

x-pack/plugin/sql/qa/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcFrozenCsvSpecIT.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.sql.qa.single_node;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvSpecTestCase;
+import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser;
+
+public class JdbcFrozenCsvSpecIT extends CsvSpecTestCase {
+
+    @ParametersFactory(argumentFormatting = PARAM_FORMATTING)
+    public static List<Object[]> readScriptSpec() throws Exception {
+        return readScriptSpec("/slow/frozen.csv-spec", specParser());
+    }
+
+    @Override
+    protected Properties connectionProperties() {
+        Properties props = new Properties(super.connectionProperties());
+        String timeout = String.valueOf(TimeUnit.MINUTES.toMillis(5));
+        props.setProperty("connect.timeout", timeout);
+        props.setProperty("network.timeout", timeout);
+        props.setProperty("query.timeout", timeout);
+        props.setProperty("page.timeout", timeout);
+
+        return props;
+    }
+
+
+    public JdbcFrozenCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
+        super(fileName, groupName, testName, lineNumber, testCase);
+    }
+}

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java

@@ -6,6 +6,7 @@
 package org.elasticsearch.xpack.sql.qa.jdbc;
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;
 
@@ -42,14 +43,8 @@ public CsvSpecTestCase(String fileName, String groupName, String testName, Integ
     protected final void doTest() throws Throwable {
         // Run the time tests always in UTC
         // TODO: https://github.com/elastic/elasticsearch/issues/40779
-        if ("time".equals(groupName)) {
-            try (Connection csv = csvConnection(testCase); Connection es = esJdbc(connectionProperties())) {
-                executeAndAssert(csv, es);
-            }
-        } else {
-            try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
-                executeAndAssert(csv, es);
-            }
+        try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
+            executeAndAssert(csv, es);
         }
     }
 

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java

@@ -49,6 +49,9 @@ protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception {
         loadLogsDatasetIntoEs(client, "logs", "logs");
         makeAlias(client, "test_alias", "test_emp", "test_emp_copy");
         makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy");
+        // frozen index
+        loadEmpDatasetIntoEs(client, "frozen_emp", "employees");
+        freeze(client, "frozen_emp");
     }
 
     public static void loadDocsDatasetIntoEs(RestClient client) throws Exception {
@@ -292,6 +295,12 @@ public static void makeAlias(RestClient client, String aliasName, String... indi
         }
     }
 
+    protected static void freeze(RestClient client, String... indices) throws Exception {
+        for (String index : indices) {
+            client.performRequest(new Request("POST", "/" + index + "/_freeze"));
+        }
+    }
+
     private static void csvToLines(String name, CheckedBiConsumer<List<String>, List<String>, Exception> consumeLine) throws Exception {
         String location = "/" + name + ".csv";
         URL dataSet = SqlSpecTestCase.class.getResource(location);

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ShowTablesTestCase.java

@@ -30,7 +30,7 @@ public void testShowTablesWithManyIndices() throws Exception {
             for (int i = 0; i < indices; i++) {
                 String index = String.format(Locale.ROOT, "test%02d", i);
                 index(index, builder -> builder.field("name", "bob"));
-                h2.createStatement().executeUpdate("INSERT INTO mock VALUES ('" + index + "', 'BASE TABLE');");
+                h2.createStatement().executeUpdate("INSERT INTO mock VALUES ('" + index + "', 'BASE TABLE', 'INDEX');");
             }
 
             ResultSet expected = h2.createStatement().executeQuery("SELECT * FROM mock ORDER BY name");

x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec

@@ -82,20 +82,20 @@ salary              |INTEGER        |integer
 showAlias
 SHOW TABLES LIKE 'test\_alias' ESCAPE '\';
 
-name:s               | type:s
+name:s               | type:s  | kind:s
 
-test_alias           | VIEW
+test_alias           | VIEW    | ALIAS
 ;
 
 showPattern
 SHOW TABLES LIKE 'test_%';
 
-name:s               | type:s
+name:s               | type:s      | kind :s
 
-test_alias           | VIEW
-test_alias_emp       | VIEW
-test_emp             | BASE TABLE
-test_emp_copy        | BASE TABLE
+test_alias           | VIEW        | ALIAS
+test_alias_emp       | VIEW        | ALIAS
+test_emp             | BASE TABLE  | INDEX
+test_emp_copy        | BASE TABLE  | INDEX
 ;
 
 groupByOnAlias

x-pack/plugin/sql/qa/src/main/resources/command.csv-spec

@@ -198,49 +198,49 @@ TODAY          |SCALAR
 showTables
 SHOW TABLES;
 
-       name        |     type      
-logs               |BASE TABLE
-test_alias         |VIEW          
-test_alias_emp     |VIEW          
-test_emp           |BASE TABLE     
-test_emp_copy      |BASE TABLE     
+       name    |     type   |  kind   
+logs           |BASE TABLE  |INDEX
+test_alias     |VIEW        |ALIAS
+test_alias_emp |VIEW        |ALIAS 
+test_emp       |BASE TABLE  |INDEX  
+test_emp_copy  |BASE TABLE  |INDEX  
 ;
 
 showTablesSimpleLike
 SHOW TABLES LIKE 'test_emp';
 
- name:s        | type:s
-test_emp       |BASE TABLE
+ name:s        | type:s     | kind:s
+test_emp       |BASE TABLE  |INDEX
 ;
 
 showTablesMultiLike
 SHOW TABLES LIKE 'test_emp%';
 
- name:s            | type:s
-test_emp           |BASE TABLE     
-test_emp_copy      |BASE TABLE     
+ name:s        | type:s    |kind:s
+test_emp       |BASE TABLE |INDEX     
+test_emp_copy  |BASE TABLE |INDEX     
 ;
 
 showTablesIdentifier
 SHOW TABLES "test_emp";
 
- name:s        | type:s
-test_emp       |BASE TABLE
+ name:s        | type:s     |kind:s
+test_emp       |BASE TABLE  |INDEX
 ;
 
 showTablesIdentifierPattern
 SHOW TABLES "test_e*,-test_emp";
 
- name:s            | type:s
-test_emp_copy      |BASE TABLE
+ name:s        | type:s    |kind:s
+test_emp_copy  |BASE TABLE |INDEX
 ;
 
 showTablesIdentifierPatternOnAliases
 SHOW TABLES "test*,-test_emp*";
 
- name:s        | type:s
-test_alias     |VIEW          
-test_alias_emp |VIEW          
+ name:s        | type:s | kind:s
+test_alias     |VIEW    |ALIAS          
+test_alias_emp |VIEW    |ALIAS          
 ;
 
 // DESCRIBE

x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec

@@ -93,11 +93,11 @@ showTables
 // tag::showTables
 SHOW TABLES;
 
-     name      |     type      
----------------+---------------
-emp            |BASE TABLE     
-employees      |VIEW          
-library        |BASE TABLE 
+     name      |     type      |     kind      
+---------------+---------------+---------------
+emp            |BASE TABLE     |INDEX          
+employees      |VIEW           |ALIAS          
+library        |BASE TABLE     |INDEX    
 
 // end::showTables
 ;
@@ -106,9 +106,9 @@ showTablesLikeExact
 // tag::showTablesLikeExact
 SHOW TABLES LIKE 'emp';
 
-     name      |     type      
----------------+---------------
-emp            |BASE TABLE     
+     name      |     type      |     kind      
+---------------+---------------+---------------
+emp            |BASE TABLE     |INDEX    
 
 // end::showTablesLikeExact
 ;
@@ -117,10 +117,10 @@ showTablesLikeWildcard
 // tag::showTablesLikeWildcard
 SHOW TABLES LIKE 'emp%';
 
-     name      |     type      
----------------+---------------
-emp            |BASE TABLE     
-employees      |VIEW          
+     name      |     type      |     kind      
+---------------+---------------+---------------
+emp            |BASE TABLE     |INDEX          
+employees      |VIEW           |ALIAS             
 
 // end::showTablesLikeWildcard
 ;
@@ -130,9 +130,9 @@ showTablesLikeOneChar
 // tag::showTablesLikeOneChar
 SHOW TABLES LIKE 'em_';
 
-     name      |     type      
----------------+---------------
-emp            |BASE TABLE     
+     name      |     type      |     kind      
+---------------+---------------+---------------
+emp            |BASE TABLE     |INDEX     
 
 // end::showTablesLikeOneChar
 ;
@@ -141,20 +141,20 @@ showTablesLikeMixed
 // tag::showTablesLikeMixed
 SHOW TABLES LIKE '%em_';
 
-     name      |     type      
----------------+---------------
-emp            |BASE TABLE     
+     name      |     type      |     kind      
+---------------+---------------+---------------
+emp            |BASE TABLE     |INDEX        
 
 // end::showTablesLikeMixed
 ;
 
 showTablesLikeEscape
-schema::name:s|type:s
+schema::name:s|type:s|kind:s
 // tag::showTablesLikeEscape
 SHOW TABLES LIKE 'emp!%' ESCAPE '!';
 
-     name      |     type      
----------------+---------------
+     name      |     type      |     kind      
+---------------+---------------+---------------
 
 // end::showTablesLikeEscape
 ;
@@ -164,10 +164,10 @@ showTablesEsMultiIndex
 // tag::showTablesEsMultiIndex
 SHOW TABLES "*,-l*";
 
-     name      |     type      
----------------+---------------
-emp            |BASE TABLE     
-employees      |VIEW
+     name      |     type      |     kind      
+---------------+---------------+---------------
+emp            |BASE TABLE     |INDEX          
+employees      |VIEW           |ALIAS  
 
 // end::showTablesEsMultiIndex
 ;

x-pack/plugin/sql/qa/src/main/resources/setup_mock_show_tables.sql

@@ -1,4 +1,5 @@
 CREATE TABLE mock (
   "name" VARCHAR,
-  "type" VARCHAR
+  "type" VARCHAR,
+  "kind" VARCHAR
 );