Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Merge develop to master #553

Merged
merged 49 commits into from
Jul 9, 2020
Merged
Changes from 1 commit
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
9f0d57a
[PPL] init basic codebase structure (#409)
penghuo Apr 6, 2020
13f8b85
[PPL] add basic parser support (#415)
penghuo Apr 8, 2020
fb4ab0b
Create ANTLR grammar files for PPL - Phase0 (#421)
chloe-zh Apr 9, 2020
aabb053
add the support of expression value (#432)
penghuo Apr 20, 2020
54049bd
Add support of Resolved Expression (#436)
penghuo Apr 23, 2020
ded0421
[PPL] Create ANTLR parser and implement basic syntax - Phase0 (#429)
chloe-zh Apr 23, 2020
d4147a9
change the package from plans to ast (#438)
penghuo Apr 24, 2020
242ca1f
[PPL] Implemeted full functionality of commands in phase0 (#440)
chloe-zh May 7, 2020
e7e2ecb
Feature/analyzer (#464)
penghuo May 11, 2020
ce2c08b
Implement get index mapping in ES storage engine (#469)
dai-chen May 18, 2020
30703ea
Add where stats rename command (#466)
penghuo May 18, 2020
aa890bc
[PPL] Added arithmetic binary operations in ppl parser (#474)
chloe-zh May 20, 2020
5ce4b56
[PPL] Add simple json formatter (#476)
dai-chen May 20, 2020
8e4453f
[PPL] Add support for fields command (#472)
penghuo May 20, 2020
d265e19
[PPL] Add Elasticsearch index scan operator (#475)
dai-chen May 22, 2020
9498e57
[PPL] Added PPL integration test base (#479)
chloe-zh May 22, 2020
8274edd
[PPL] Support eval command (#477)
penghuo May 22, 2020
3c3cb85
[PPL] Add Elasticsearch executor and integrate all components (#482)
dai-chen May 22, 2020
3aa0fec
[PPL] Support sort command (#483)
penghuo May 22, 2020
83e9b75
[PPL] Support dedup command (#485)
penghuo May 26, 2020
9ac0819
[PPL] Add the analyzer constructor (#487)
penghuo May 26, 2020
3a740ad
[PPL] Add ES rest client to support standalone mode (#484)
dai-chen May 26, 2020
4d226ff
Revert "[PPL] Add ES rest client to support standalone mode (#484)" (…
dai-chen May 26, 2020
5db0412
[PPL] Add ES rest client to support standalone mode (#491)
dai-chen May 26, 2020
5ae2203
Fix json formatter for consistency with SQL (#492)
dai-chen May 27, 2020
1c16f2f
[PPL] Support Count aggregator and OR operator (#493)
penghuo May 28, 2020
0f3f3f8
[PPL] Added integration tests for query analysis and search, where, f…
chloe-zh May 29, 2020
d9c05c5
[Bug] Support duplicated event for sort command (#499)
penghuo Jun 1, 2020
d81029d
[PPL] Add Doctest module (#497)
zhongnansu Jun 1, 2020
386da03
Added integration tests for dedup, rename, sort, stats commands (#498)
chloe-zh Jun 1, 2020
a261188
Merge branch 'master' into feature/ppl-merge-master
dai-chen Jun 1, 2020
d5f0e96
Enfore google java style (#502)
penghuo Jun 3, 2020
64e8562
Move old SQL code to legacy module (#506)
dai-chen Jun 4, 2020
3186242
[Bug Fix] Remove the GET request description in the SQL doc (#511)
penghuo Jun 11, 2020
c8fdbeb
Enable to rename multiple fields in syntax (#514)
chloe-zh Jun 11, 2020
7d53d82
[PPL] Add PPL documentation. (#516)
penghuo Jun 12, 2020
2a217ed
[PPL] Added syntax check exception respect to syntax analysis errors …
chloe-zh Jun 18, 2020
88a9c5d
[PPL] Support expression in syntax (#524)
chloe-zh Jun 22, 2020
dc02438
Bug fix, support long type for aggregation (#529)
penghuo Jun 22, 2020
d42dcbd
Move the index mapping definitions to standalone json files (#526)
pakio Jun 23, 2020
0fad590
[PPL] Support expression and boolean operators (#518)
chloe-zh Jun 26, 2020
087da88
Add new sql module (#527)
dai-chen Jun 26, 2020
df16ffe
[PPL] Add Resource monitor to avoid OOM (#533)
penghuo Jun 29, 2020
d60997b
Enforce comparison test in new SQL module (#536)
dai-chen Jul 1, 2020
05d3699
Support LIKE operator (#534)
chloe-zh Jul 8, 2020
dddef2e
Add select expression without query support (#542)
dai-chen Jul 8, 2020
bd379cf
Add plugin setting for new SQL module (#544)
dai-chen Jul 8, 2020
8df9a88
Redefine the Expression interface and Type (#550)
penghuo Jul 8, 2020
785f623
Bump develop to 1.9, fix all build errors (#552)
joshuali925 Jul 9, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[PPL] add basic parser support (#415)
* add basic parser support

* Reduce the permission by removing java.security.AllPermission
  • Loading branch information
penghuo authored Apr 8, 2020
commit 13f8b85ae00269d2fd673ac330a0d336d2e5627a
11 changes: 0 additions & 11 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -56,18 +56,7 @@ allprojects {

apply plugin: 'elasticsearch.esplugin'
apply plugin: 'elasticsearch.testclusters'
// Todo, removed after migrate SQL to subProject
project('plugin') {
apply plugin: 'elasticsearch.esplugin'
}
// Todo, removed after migrate SQL to subProject
project('integ-test') {
apply plugin: 'elasticsearch.testclusters'
}
apply plugin: 'jacoco'
if (!System.properties.containsKey('tests.rest.cluster') && !System.properties.containsKey('tests.cluster')){
apply from: 'build-tools/sqlplugin-coverage.gradle'
}
apply plugin: 'antlr'

jacoco {
14 changes: 14 additions & 0 deletions common/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
plugins {
id 'java'
id "io.freefair.lombok"
}

repositories {
mavenCentral()
}

dependencies {
compile "org.antlr:antlr4-runtime:4.7.1"

testCompile group: 'junit', name: 'junit', version: '4.12'
}
2 changes: 2 additions & 0 deletions common/lombok.config
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# This file is generated by the 'io.freefair.lombok' Gradle plugin
config.stopBubbling = true
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.sql.common.antlr;

import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.misc.Interval;

/**
* Custom stream to convert character to upper case for case insensitive grammar before sending to lexer.
*/
public class CaseInsensitiveCharStream implements CharStream {

/** Character stream */
private final CharStream charStream;

public CaseInsensitiveCharStream(String sql) {
this.charStream = CharStreams.fromString(sql);
}

@Override
public String getText(Interval interval) {
return charStream.getText(interval);
}

@Override
public void consume() {
charStream.consume();
}

@Override
public int LA(int i) {
int c = charStream.LA(i);
if (c <= 0) {
return c;
}
return Character.toUpperCase(c);
}

@Override
public int mark() {
return charStream.mark();
}

@Override
public void release(int marker) {
charStream.release(marker);
}

@Override
public int index() {
return charStream.index();
}

@Override
public void seek(int index) {
charStream.seek(index);
}

@Override
public int size() {
return charStream.size();
}

@Override
public String getSourceName() {
return charStream.getSourceName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.sql.common.antlr;

import org.antlr.v4.runtime.BaseErrorListener;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.RecognitionException;
import org.antlr.v4.runtime.Recognizer;
import org.antlr.v4.runtime.Token;
import org.antlr.v4.runtime.misc.IntervalSet;

import java.util.Locale;

/**
* Syntax analysis error listener that handles any syntax error by throwing exception with useful information.
*/
public class SyntaxAnalysisErrorListener extends BaseErrorListener {

@Override
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol,
int line, int charPositionInLine, String msg,
RecognitionException e) {

CommonTokenStream tokens = (CommonTokenStream) recognizer.getInputStream();
Token offendingToken = (Token) offendingSymbol;
String query = tokens.getText();

throw new RuntimeException(
String.format(Locale.ROOT,
"Failed to parse query due to offending symbol [%s] at: '%s' <--- HERE... More details: %s",
getOffendingText(offendingToken),
truncateQueryAtOffendingToken(query, offendingToken),
getDetails(recognizer, msg, e)
)
);
}

private String getOffendingText(Token offendingToken) {
return offendingToken.getText();
}

private String truncateQueryAtOffendingToken(String query, Token offendingToken) {
return query.substring(0, offendingToken.getStopIndex() + 1);
}

/**
* As official JavaDoc says, e=null means parser was able to recover from the error.
* In other words, "msg" argument includes the information we want.
*/
private String getDetails(Recognizer<?, ?> recognizer, String msg, RecognitionException e) {
String details;
if (e == null) {
details = msg;
} else {
IntervalSet followSet = e.getExpectedTokens();
details = "Expecting tokens in " + followSet.toString(recognizer.getVocabulary());
}
return details;
}

}
10 changes: 10 additions & 0 deletions elasticsearch/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
apply plugin: 'java'

repositories {
mavenCentral()
}

dependencies {
compile group: 'org.elasticsearch', name: 'elasticsearch', version: "${es_version}"
testCompile group: 'junit', name: 'junit', version: '4.12'
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.sql.elasticsearch.security;

import org.elasticsearch.SpecialPermission;

import java.io.IOException;
import java.security.AccessController;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;

/**
* Ref: https://www.elastic.co/guide/en/elasticsearch/plugins/current/plugin-authors.html#_java_security_permissions
*/
public class SecurityAccess {

public static <T> T doPrivileged(final PrivilegedExceptionAction<T> operation) throws IOException {
SpecialPermission.check();
try {
return AccessController.doPrivileged(operation);
} catch (final PrivilegedActionException e) {
throw (IOException) e.getCause();
}
}
}
1 change: 1 addition & 0 deletions integ-test/build.gradle
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@ dependencies {
dependencyLicenses.enabled = false
testingConventions.enabled = false
checkstyleTest.ignoreFailures = true
forbiddenApisTest.enabled = false

tasks.integTest.dependsOn(':plugin:bundlePlugin')
testClusters.integTest {
Original file line number Diff line number Diff line change
@@ -17,16 +17,57 @@

import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.test.rest.ESRestTestCase;
import org.hamcrest.Description;
import org.hamcrest.TypeSafeMatcher;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;

import java.io.IOException;
import java.util.Locale;

import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.hasProperty;

public class PPLPluginIT extends ESRestTestCase {
@Rule
public ExpectedException exceptionRule = ExpectedException.none();

@Test
public void testQueryEndpointShouldOK() throws IOException {
Response response = client().performRequest(new Request("POST", "/_opendistro/_ppl"));
assertThat(response.getStatusLine().getStatusCode(), is(200));
Response response = client().performRequest(makeRequest("search source=a"));
assertThat(response, statusCode(200));
}

@Test
public void testQueryEndpointShouldFail() throws IOException {
exceptionRule.expect(ResponseException.class);
exceptionRule.expect(hasProperty("response", statusCode(500)));

client().performRequest(makeRequest("search invalid"));
}

protected Request makeRequest(String query) {
Request post = new Request("POST", "/_opendistro/_ppl");
post.setJsonEntity(String.format(Locale.ROOT,
"{\n" +
" \"query\": \"%s\"\n" +
"}", query));
return post;
}

private TypeSafeMatcher<Response> statusCode(int statusCode) {
return new TypeSafeMatcher<Response>() {
@Override
public void describeTo(Description description) {
description.appendText(String.format(Locale.ROOT, "statusCode=%d", statusCode));
}

@Override
protected boolean matchesSafely(Response resp) {
return resp.getStatusLine().getStatusCode() == statusCode;
}
};
}
}
9 changes: 9 additions & 0 deletions plugin/build.gradle
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
apply plugin: 'java'
apply plugin: 'idea'
apply plugin: 'elasticsearch.esplugin'

ext {
projectSubstitutions = [:]
@@ -20,7 +21,15 @@ esplugin {
test.enabled = false
integTest.enabled = false
dependencyLicenses.enabled = false
thirdPartyAudit.enabled = false

configurations.all {
// conflict with spring-jcl
exclude group: "commons-logging", module: "commons-logging"
}

dependencies {
compile group: 'org.springframework', name: 'spring-beans', version: '5.2.5.RELEASE'
compile project(":ppl")
compile project(':elasticsearch')
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.sql.plugin.request;

import com.amazon.opendistroforelasticsearch.sql.ppl.domain.PPLQueryRequest;
import org.elasticsearch.rest.RestRequest;
import org.json.JSONException;
import org.json.JSONObject;

public class PPLQueryRequestFactory {
private static final String PPL_URL_PARAM_KEY = "ppl";
private static final String PPL_FIELD_NAME = "query";

public static PPLQueryRequest getPPLRequest(RestRequest request) {
switch (request.method()) {
case GET:
return parsePPLRequestFromUrl(request);
case POST:
return parsePPLRequestFromPayload(request);
default:
throw new IllegalArgumentException("ES PPL doesn't supported HTTP " + request.method().name());
}
}

private static PPLQueryRequest parsePPLRequestFromUrl(RestRequest restRequest) {
String ppl;

ppl = restRequest.param(PPL_URL_PARAM_KEY);
if (ppl == null) {
throw new IllegalArgumentException("Cannot find ppl parameter from the URL");
}
return new PPLQueryRequest(ppl, null);
}

private static PPLQueryRequest parsePPLRequestFromPayload(RestRequest restRequest) {
String content = restRequest.content().utf8ToString();
JSONObject jsonContent;
try {
jsonContent = new JSONObject(content);
} catch (JSONException e) {
throw new IllegalArgumentException("Failed to parse request payload", e);
}
return new PPLQueryRequest(jsonContent.getString(PPL_FIELD_NAME), jsonContent);
}
}
Loading