Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SQL: Add support for FROZEN indices #41558

Merged
merged 11 commits into from
May 9, 2019
Merged

SQL: Add support for FROZEN indices #41558

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
b6a63ea
SQL: Add support for FROZEN indices
costin Feb 27, 2019
dd822d6
Consider frozen indices inside SysTables
costin Apr 26, 2019
f56649a
Merge remote-tracking branch 'remotes/upstream/master' into sql/froze…
costin May 3, 2019
e1c02f2
Address feedback
costin May 3, 2019
e0334e4
update tests
costin May 3, 2019
6103d9b
Merge remote-tracking branch 'remotes/upstream/master' into sql/froze…
costin May 3, 2019
7bddf5c
Move frozen to dedicated folder
costin May 3, 2019
cf63d3c
Address feedback
costin May 6, 2019
9d3be49
Merge branch 'master' into sql/frozen-indices
costin May 6, 2019
8ac59f8
Merge branch 'master' into sql/frozen-indices
costin May 9, 2019
ddf5509
Update test
costin May 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcConfiguration.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,10 +57,13 @@ class JdbcConfiguration extends ConnectionConfiguration {
static final String FIELD_MULTI_VALUE_LENIENCY = "field.multi.value.leniency";
static final String FIELD_MULTI_VALUE_LENIENCY_DEFAULT = "true";

static final String INDEX_INCLUDE_FROZEN = "index.include.frozen";
static final String INDEX_INCLUDE_FROZEN_DEFAULT = "false";


// options that don't change at runtime
private static final Set<String> OPTION_NAMES = new LinkedHashSet<>(
Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, DEBUG, DEBUG_OUTPUT));
Arrays.asList(TIME_ZONE, FIELD_MULTI_VALUE_LENIENCY, INDEX_INCLUDE_FROZEN, DEBUG, DEBUG_OUTPUT));

static {
// trigger version initialization
Expand All @@ -77,6 +80,7 @@ class JdbcConfiguration extends ConnectionConfiguration {
// mutable ones
private ZoneId zoneId;
private boolean fieldMultiValueLeniency;
private boolean includeFrozen;

public static JdbcConfiguration create(String u, Properties props, int loginTimeoutSeconds) throws JdbcSQLException {
URI uri = parseUrl(u);
Expand Down Expand Up @@ -159,6 +163,8 @@ private JdbcConfiguration(URI baseURI, String u, Properties props) throws JdbcSQ
s -> TimeZone.getTimeZone(s).toZoneId().normalized());
this.fieldMultiValueLeniency = parseValue(FIELD_MULTI_VALUE_LENIENCY,
props.getProperty(FIELD_MULTI_VALUE_LENIENCY, FIELD_MULTI_VALUE_LENIENCY_DEFAULT), Boolean::parseBoolean);
this.includeFrozen = parseValue(INDEX_INCLUDE_FROZEN, props.getProperty(INDEX_INCLUDE_FROZEN, INDEX_INCLUDE_FROZEN_DEFAULT),
Boolean::parseBoolean);
}

@Override
Expand Down Expand Up @@ -186,6 +192,10 @@ public boolean fieldMultiValueLeniency() {
return fieldMultiValueLeniency;
}

public boolean indexIncludeFrozen() {
return includeFrozen;
}

public static boolean canAccept(String url) {
return (StringUtils.hasText(url) && url.trim().startsWith(JdbcConfiguration.URL_PREFIX));
}
Expand Down
3 changes: 2 additions & 1 deletion x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcHttpClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ Cursor query(String sql, List<SqlTypedParamValue> params, RequestMeta meta) thro
Boolean.FALSE,
null,
new RequestInfo(Mode.JDBC),
conCfg.fieldMultiValueLeniency());
conCfg.fieldMultiValueLeniency(),
conCfg.indexIncludeFrozen());
SqlQueryResponse response = httpClient.query(sqlRequest);
return new DefaultCursor(this, response.cursor(), toJdbcColumnInfo(response.columns()), response.rows(), meta);
}
Expand Down
3 changes: 2 additions & 1 deletion .../sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/CliSecurityIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ public void expectShowTables(List<String> tables, String user) throws Exception
String tablesOutput = cli.command("SHOW TABLES");
assertThat(tablesOutput, containsString("name"));
assertThat(tablesOutput, containsString("type"));
assertEquals("---------------+---------------", cli.readLine());
assertThat(tablesOutput, containsString("kind"));
assertEquals("---------------+---------------+---------------", cli.readLine());
for (String table : tables) {
String line = null;
/*
Expand Down
2 changes: 2 additions & 0 deletions .../qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/RestSqlSecurityIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,15 @@ public void expectShowTables(List<String> tables, String user) throws Exception
List<Object> columns = new ArrayList<>();
columns.add(columnInfo(mode, "name", "keyword", JDBCType.VARCHAR, 32766));
columns.add(columnInfo(mode, "type", "keyword", JDBCType.VARCHAR, 32766));
columns.add(columnInfo(mode, "kind", "keyword", JDBCType.VARCHAR, 32766));
Map<String, Object> expected = new HashMap<>();
expected.put("columns", columns);
List<List<String>> rows = new ArrayList<>();
for (String table : tables) {
List<String> fields = new ArrayList<>();
fields.add(table);
fields.add("BASE TABLE");
fields.add("INDEX");
rows.add(fields);
}
expected.put("rows", rows);
Expand Down
4 changes: 2 additions & 2 deletions ...a/single-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcCsvSpecIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,12 @@

public class JdbcCsvSpecIT extends CsvSpecTestCase {


@ParametersFactory(argumentFormatting = PARAM_FORMATTING)
public static List<Object[]> readScriptSpec() throws Exception {
List<Object[]> list = new ArrayList<>();
list.addAll(CsvSpecTestCase.readScriptSpec());
return readScriptSpec("/single-node-only/command-sys.csv-spec", specParser());
list.addAll(readScriptSpec("/single-node-only/command-sys.csv-spec", specParser()));
return list;
}

public JdbcCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
Expand Down
42 changes: 42 additions & 0 deletions ...le-node/src/test/java/org/elasticsearch/xpack/sql/qa/single_node/JdbcFrozenCsvSpecIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.xpack.sql.qa.single_node;

import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.elasticsearch.xpack.sql.qa.jdbc.CsvSpecTestCase;
import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;

import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

import static org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.specParser;

public class JdbcFrozenCsvSpecIT extends CsvSpecTestCase {

@ParametersFactory(argumentFormatting = PARAM_FORMATTING)
public static List<Object[]> readScriptSpec() throws Exception {
return readScriptSpec("/slow/frozen.csv-spec", specParser());
}

@Override
protected Properties connectionProperties() {
Properties props = new Properties(super.connectionProperties());
String timeout = String.valueOf(TimeUnit.MINUTES.toMillis(5));
props.setProperty("connect.timeout", timeout);
props.setProperty("network.timeout", timeout);
props.setProperty("query.timeout", timeout);
props.setProperty("page.timeout", timeout);

return props;
}


public JdbcFrozenCsvSpecIT(String fileName, String groupName, String testName, Integer lineNumber, CsvTestCase testCase) {
super(fileName, groupName, testName, lineNumber, testCase);
}
}
11 changes: 3 additions & 8 deletions x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
package org.elasticsearch.xpack.sql.qa.jdbc;

import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.apache.logging.log4j.Logger;
import org.elasticsearch.xpack.sql.qa.jdbc.CsvTestUtils.CsvTestCase;

Expand Down Expand Up @@ -42,14 +43,8 @@ public CsvSpecTestCase(String fileName, String groupName, String testName, Integ
protected final void doTest() throws Throwable {
// Run the time tests always in UTC
// TODO: https://github.com/elastic/elasticsearch/issues/40779
if ("time".equals(groupName)) {
try (Connection csv = csvConnection(testCase); Connection es = esJdbc(connectionProperties())) {
executeAndAssert(csv, es);
}
} else {
try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
executeAndAssert(csv, es);
}
try (Connection csv = csvConnection(testCase); Connection es = esJdbc()) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@matriv The previous hack was unnecessary - the conditional should be addressed in connectionProperties and further more, esJdbc() actually calls esJdbc(connectionProperties()) so the code was not only duplicated but also identical in results.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't get how this is addressed only for a specific file, in this case: time.csv-spec.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is not but then again, it wasn't before.
In other words, the if ("time".equals(groupName)) does the right thing but calling esJdbc(connectionProperties()) is the same as calling esJdbc() meaning the if and else branch end up calling the same code path.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not the case, if you check again there are 2 different connectionProperties() involved there. and the one for the time sets the timezone to UTC where the other uses a random timezone.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@costin Did you check this ^^ ?

executeAndAssert(csv, es);
}
}

Expand Down
9 changes: 9 additions & 0 deletions x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/DataLoader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@ protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception {
loadLogsDatasetIntoEs(client, "logs", "logs");
makeAlias(client, "test_alias", "test_emp", "test_emp_copy");
makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy");
// frozen index
loadEmpDatasetIntoEs(client, "frozen_emp", "employees");
freeze(client, "frozen_emp");
}

public static void loadDocsDatasetIntoEs(RestClient client) throws Exception {
Expand Down Expand Up @@ -292,6 +295,12 @@ protected static void makeAlias(RestClient client, String aliasName, String... i
}
}

protected static void freeze(RestClient client, String... indices) throws Exception {
for (String index : indices) {
client.performRequest(new Request("POST", "/" + index + "/_freeze"));
}
}

private static void csvToLines(String name, CheckedBiConsumer<List<String>, List<String>, Exception> consumeLine) throws Exception {
String location = "/" + name + ".csv";
URL dataSet = SqlSpecTestCase.class.getResource(location);
Expand Down
2 changes: 1 addition & 1 deletion ...k/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/ShowTablesTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public void testShowTablesWithManyIndices() throws Exception {
for (int i = 0; i < indices; i++) {
String index = String.format(Locale.ROOT, "test%02d", i);
index(index, builder -> builder.field("name", "bob"));
h2.createStatement().executeUpdate("INSERT INTO mock VALUES ('" + index + "', 'BASE TABLE');");
h2.createStatement().executeUpdate("INSERT INTO mock VALUES ('" + index + "', 'BASE TABLE', 'INDEX');");
}

ResultSet expected = h2.createStatement().executeQuery("SELECT * FROM mock ORDER BY name");
Expand Down
14 changes: 7 additions & 7 deletions x-pack/plugin/sql/qa/src/main/resources/alias.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,20 +82,20 @@ salary |INTEGER |integer
showAlias
SHOW TABLES LIKE 'test\_alias' ESCAPE '\';

name:s | type:s
name:s | type:s | kind:s

test_alias | VIEW
test_alias | VIEW | INDEX
;

showPattern
SHOW TABLES LIKE 'test_%';

name:s | type:s
name:s | type:s | kind :s

test_alias | VIEW
test_alias_emp | VIEW
test_emp | BASE TABLE
test_emp_copy | BASE TABLE
test_alias | VIEW | ALIAS
test_alias_emp | VIEW | ALIAS
test_emp | BASE TABLE | INDEX
test_emp_copy | BASE TABLE | INDEX
;

groupByOnAlias
Expand Down
36 changes: 18 additions & 18 deletions x-pack/plugin/sql/qa/src/main/resources/command.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,49 +189,49 @@ TODAY |SCALAR
showTables
SHOW TABLES;

name | type
logs |BASE TABLE
test_alias |VIEW
test_alias_emp |VIEW
test_emp |BASE TABLE
test_emp_copy |BASE TABLE
name | type | kind
logs |BASE TABLE |INDEX
test_alias |VIEW |ALIAS
test_alias_emp |VIEW |ALIAS
test_emp |BASE TABLE |INDEX
test_emp_copy |BASE TABLE |INDEX
;

showTablesSimpleLike
SHOW TABLES LIKE 'test_emp';

name:s | type:s
test_emp |BASE TABLE
name:s | type:s | kind:s
test_emp |BASE TABLE |INDEX
;

showTablesMultiLike
SHOW TABLES LIKE 'test_emp%';

name:s | type:s
test_emp |BASE TABLE
test_emp_copy |BASE TABLE
name:s | type:s |kind:s
test_emp |BASE TABLE |INDEX
test_emp_copy |BASE TABLE |INDEX
;

showTablesIdentifier
SHOW TABLES "test_emp";

name:s | type:s
test_emp |BASE TABLE
name:s | type:s |kind:s
test_emp |BASE TABLE |INDEX
;

showTablesIdentifierPattern
SHOW TABLES "test_e*,-test_emp";

name:s | type:s
test_emp_copy |BASE TABLE
name:s | type:s |kind:s
test_emp_copy |BASE TABLE |INDEX
;

showTablesIdentifierPatternOnAliases
SHOW TABLES "test*,-test_emp*";

name:s | type:s
test_alias |VIEW
test_alias_emp |VIEW
name:s | type:s | kind:s
test_alias |VIEW |ALIAS
test_alias_emp |VIEW |ALIAS
;

// DESCRIBE
Expand Down
50 changes: 25 additions & 25 deletions x-pack/plugin/sql/qa/src/main/resources/docs/docs.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,11 @@ showTables
// tag::showTables
SHOW TABLES;

name | type
---------------+---------------
emp |BASE TABLE
employees |VIEW
library |BASE TABLE
name | type | kind
---------------+---------------+---------------
emp |BASE TABLE |INDEX
employees |VIEW |ALIAS
library |BASE TABLE |INDEX

// end::showTables
;
Expand All @@ -106,9 +106,9 @@ showTablesLikeExact
// tag::showTablesLikeExact
SHOW TABLES LIKE 'emp';

name | type
---------------+---------------
emp |BASE TABLE
name | type | kind
---------------+---------------+---------------
emp |BASE TABLE |INDEX

// end::showTablesLikeExact
;
Expand All @@ -117,10 +117,10 @@ showTablesLikeWildcard
// tag::showTablesLikeWildcard
SHOW TABLES LIKE 'emp%';

name | type
---------------+---------------
emp |BASE TABLE
employees |VIEW
name | type | kind
---------------+---------------+---------------
emp |BASE TABLE |INDEX
employees |VIEW |ALIAS

// end::showTablesLikeWildcard
;
Expand All @@ -130,9 +130,9 @@ showTablesLikeOneChar
// tag::showTablesLikeOneChar
SHOW TABLES LIKE 'em_';

name | type
---------------+---------------
emp |BASE TABLE
name | type | kind
---------------+---------------+---------------
emp |BASE TABLE |INDEX

// end::showTablesLikeOneChar
;
Expand All @@ -141,20 +141,20 @@ showTablesLikeMixed
// tag::showTablesLikeMixed
SHOW TABLES LIKE '%em_';

name | type
---------------+---------------
emp |BASE TABLE
name | type | kind
---------------+---------------+---------------
emp |BASE TABLE |INDEX

// end::showTablesLikeMixed
;

showTablesLikeEscape
schema::name:s|type:s
schema::name:s|type:s|kind:s
// tag::showTablesLikeEscape
SHOW TABLES LIKE 'emp!%' ESCAPE '!';

name | type
---------------+---------------
name | type | kind
---------------+---------------+---------------

// end::showTablesLikeEscape
;
Expand All @@ -164,10 +164,10 @@ showTablesEsMultiIndex
// tag::showTablesEsMultiIndex
SHOW TABLES "*,-l*";

name | type
---------------+---------------
emp |BASE TABLE
employees |VIEW
name | type | kind
---------------+---------------+---------------
emp |BASE TABLE |INDEX
employees |VIEW |ALIAS

// end::showTablesEsMultiIndex
;
Expand Down
3 changes: 2 additions & 1 deletion x-pack/plugin/sql/qa/src/main/resources/setup_mock_show_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
CREATE TABLE mock (
"name" VARCHAR,
"type" VARCHAR
"type" VARCHAR,
"kind" VARCHAR
);
Loading