Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Merge develop to master #553

Merged
merged 49 commits into from
Jul 9, 2020
Merged
Changes from 1 commit
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
9f0d57a
[PPL] init basic codebase structure (#409)
penghuo Apr 6, 2020
13f8b85
[PPL] add basic parser support (#415)
penghuo Apr 8, 2020
fb4ab0b
Create ANTLR grammar files for PPL - Phase0 (#421)
chloe-zh Apr 9, 2020
aabb053
add the support of expression value (#432)
penghuo Apr 20, 2020
54049bd
Add support of Resolved Expression (#436)
penghuo Apr 23, 2020
ded0421
[PPL] Create ANTLR parser and implement basic syntax - Phase0 (#429)
chloe-zh Apr 23, 2020
d4147a9
change the package from plans to ast (#438)
penghuo Apr 24, 2020
242ca1f
[PPL] Implemeted full functionality of commands in phase0 (#440)
chloe-zh May 7, 2020
e7e2ecb
Feature/analyzer (#464)
penghuo May 11, 2020
ce2c08b
Implement get index mapping in ES storage engine (#469)
dai-chen May 18, 2020
30703ea
Add where stats rename command (#466)
penghuo May 18, 2020
aa890bc
[PPL] Added arithmetic binary operations in ppl parser (#474)
chloe-zh May 20, 2020
5ce4b56
[PPL] Add simple json formatter (#476)
dai-chen May 20, 2020
8e4453f
[PPL] Add support for fields command (#472)
penghuo May 20, 2020
d265e19
[PPL] Add Elasticsearch index scan operator (#475)
dai-chen May 22, 2020
9498e57
[PPL] Added PPL integration test base (#479)
chloe-zh May 22, 2020
8274edd
[PPL] Support eval command (#477)
penghuo May 22, 2020
3c3cb85
[PPL] Add Elasticsearch executor and integrate all components (#482)
dai-chen May 22, 2020
3aa0fec
[PPL] Support sort command (#483)
penghuo May 22, 2020
83e9b75
[PPL] Support dedup command (#485)
penghuo May 26, 2020
9ac0819
[PPL] Add the analyzer constructor (#487)
penghuo May 26, 2020
3a740ad
[PPL] Add ES rest client to support standalone mode (#484)
dai-chen May 26, 2020
4d226ff
Revert "[PPL] Add ES rest client to support standalone mode (#484)" (…
dai-chen May 26, 2020
5db0412
[PPL] Add ES rest client to support standalone mode (#491)
dai-chen May 26, 2020
5ae2203
Fix json formatter for consistency with SQL (#492)
dai-chen May 27, 2020
1c16f2f
[PPL] Support Count aggregator and OR operator (#493)
penghuo May 28, 2020
0f3f3f8
[PPL] Added integration tests for query analysis and search, where, f…
chloe-zh May 29, 2020
d9c05c5
[Bug] Support duplicated event for sort command (#499)
penghuo Jun 1, 2020
d81029d
[PPL] Add Doctest module (#497)
zhongnansu Jun 1, 2020
386da03
Added integration tests for dedup, rename, sort, stats commands (#498)
chloe-zh Jun 1, 2020
a261188
Merge branch 'master' into feature/ppl-merge-master
dai-chen Jun 1, 2020
d5f0e96
Enfore google java style (#502)
penghuo Jun 3, 2020
64e8562
Move old SQL code to legacy module (#506)
dai-chen Jun 4, 2020
3186242
[Bug Fix] Remove the GET request description in the SQL doc (#511)
penghuo Jun 11, 2020
c8fdbeb
Enable to rename multiple fields in syntax (#514)
chloe-zh Jun 11, 2020
7d53d82
[PPL] Add PPL documentation. (#516)
penghuo Jun 12, 2020
2a217ed
[PPL] Added syntax check exception respect to syntax analysis errors …
chloe-zh Jun 18, 2020
88a9c5d
[PPL] Support expression in syntax (#524)
chloe-zh Jun 22, 2020
dc02438
Bug fix, support long type for aggregation (#529)
penghuo Jun 22, 2020
d42dcbd
Move the index mapping definitions to standalone json files (#526)
pakio Jun 23, 2020
0fad590
[PPL] Support expression and boolean operators (#518)
chloe-zh Jun 26, 2020
087da88
Add new sql module (#527)
dai-chen Jun 26, 2020
df16ffe
[PPL] Add Resource monitor to avoid OOM (#533)
penghuo Jun 29, 2020
d60997b
Enforce comparison test in new SQL module (#536)
dai-chen Jul 1, 2020
05d3699
Support LIKE operator (#534)
chloe-zh Jul 8, 2020
dddef2e
Add select expression without query support (#542)
dai-chen Jul 8, 2020
bd379cf
Add plugin setting for new SQL module (#544)
dai-chen Jul 8, 2020
8df9a88
Redefine the Expression interface and Type (#550)
penghuo Jul 8, 2020
785f623
Bump develop to 1.9, fix all build errors (#552)
joshuali925 Jul 9, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Revert "[PPL] Add ES rest client to support standalone mode (#484)" (#…
…490)

This reverts commit 3a740ad.
dai-chen authored May 26, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 4d226ff449b215055de29a78f5ddb2bc3407dfb8
1 change: 0 additions & 1 deletion elasticsearch/build.gradle
Original file line number Diff line number Diff line change
@@ -11,7 +11,6 @@ repositories {
dependencies {
compile project(':core')
compile group: 'org.elasticsearch', name: 'elasticsearch', version: "${es_version}"
compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version:"${es_version}"

testImplementation('org.junit.jupiter:junit-jupiter:5.6.2')
testCompile group: 'org.hamcrest', name: 'hamcrest-library', version: '2.1'
Original file line number Diff line number Diff line change
@@ -42,12 +42,6 @@ public interface ElasticsearchClient {
*/
ElasticsearchResponse search(ElasticsearchRequest request);

/**
* Clean up resources related to the search request, for example scroll context.
* @param request search request
*/
void cleanup(ElasticsearchRequest request);

/**
* Schedule a task to run.
* @param task task
Original file line number Diff line number Diff line change
@@ -101,20 +101,16 @@ public ElasticsearchResponse search(ElasticsearchRequest request) {
esResponse = client.searchScroll(request.scrollRequest()).actionGet();
} else {
esResponse = client.search(request.searchRequest()).actionGet();
request.setScrollId(esResponse.getScrollId());
}
request.setScrollId(esResponse.getScrollId());

return new ElasticsearchResponse(esResponse);
}

@Override
public void cleanup(ElasticsearchRequest request) {
if (request.isScrollStarted()) {
ElasticsearchResponse response = new ElasticsearchResponse(esResponse);
if (response.isEmpty()) {
client.prepareClearScroll().
addScrollId(request.getScrollId()).
addScrollId(esResponse.getScrollId()).
get();
request.reset();
}
return response;
}

@Override

This file was deleted.

Original file line number Diff line number Diff line change
@@ -29,14 +29,10 @@
import java.util.Objects;

/**
* Elasticsearch search request. This has to be stateful because it needs to:
*
* 1) Accumulate search source builder when visiting logical plan to push down operation
* 2) Maintain scroll ID between calls to client search method
* Elasticsearch search request
*/
@EqualsAndHashCode
@RequiredArgsConstructor
@Getter
@ToString
public class ElasticsearchRequest {

@@ -60,6 +56,7 @@ public class ElasticsearchRequest {
/**
* Search request source builder.
*/
@Getter
private final SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();

/**
@@ -90,12 +87,4 @@ public SearchScrollRequest scrollRequest() {
scrollId(scrollId);
}

/**
* Reset internal state in case any stale data. However, ideally the same instance
* is not supposed to be reused across different physical plan.
*/
public void reset() {
scrollId = null;
}

}
Original file line number Diff line number Diff line change
@@ -63,8 +63,6 @@ public ElasticsearchIndexScan(ElasticsearchClient client, String indexName) {

@Override
public void open() {
super.open();

// For now pull all results immediately once open
List<ElasticsearchResponse> responses = new ArrayList<>();
ElasticsearchResponse response = client.search(request);
@@ -85,11 +83,4 @@ public ExprValue next() {
return ExprValueUtils.fromObjectValue(hits.next().getSourceAsMap());
}

@Override
public void close() {
super.close();

client.cleanup(request);
}

}
Original file line number Diff line number Diff line change
@@ -23,7 +23,6 @@
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.io.Resources;
import org.apache.lucene.search.TotalHits;
import org.elasticsearch.action.search.ClearScrollRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.cluster.ClusterState;
@@ -33,6 +32,7 @@
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.DeprecationHandler;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
@@ -43,9 +43,7 @@
import org.elasticsearch.threadpool.ThreadPool;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;

import java.io.IOException;
@@ -62,8 +60,6 @@
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

@ExtendWith(MockitoExtension.class)
@@ -160,6 +156,9 @@ public void search() {
when(scrollResponse.getScrollId()).thenReturn("scroll456");
when(scrollResponse.getHits()).thenReturn(SearchHits.empty());

// Mock clear scroll request
when(nodeClient.prepareClearScroll().addScrollId("scroll456").get()).thenReturn(null);

// Verify response for first scroll request
ElasticsearchRequest request = new ElasticsearchRequest("test");
ElasticsearchResponse response1 = client.search(request);
@@ -179,8 +178,9 @@ public void search() {
void schedule() {
ThreadPool threadPool = mock(ThreadPool.class);
when(threadPool.preserveContext(any())).then(invocation -> invocation.getArgument(0));
when(nodeClient.threadPool()).thenReturn(threadPool);

// Instantiate NodeClient because Mockito cannot mock final method threadPool()
nodeClient = new NodeClient(Settings.EMPTY, threadPool);
doAnswer(invocation -> {
Runnable task = invocation.getArgument(0);
task.run();
@@ -194,37 +194,6 @@ void schedule() {
assertTrue(isRun.get());
}

@Test
void cleanup() {
ElasticsearchNodeClient client = new ElasticsearchNodeClient(mock(ClusterService.class),
nodeClient);

ClearScrollRequestBuilder requestBuilder = mock(ClearScrollRequestBuilder.class);
when(nodeClient.prepareClearScroll()).thenReturn(requestBuilder);
when(requestBuilder.addScrollId(any())).thenReturn(requestBuilder);
when(requestBuilder.get()).thenReturn(null);

ElasticsearchRequest request = new ElasticsearchRequest("test");
request.setScrollId("scroll123");
client.cleanup(request);
assertFalse(request.isScrollStarted());

InOrder inOrder = Mockito.inOrder(nodeClient, requestBuilder);
inOrder.verify(nodeClient).prepareClearScroll();
inOrder.verify(requestBuilder).addScrollId("scroll123");
inOrder.verify(requestBuilder).get();
}

@Test
void cleanupWithoutScrollId() {
ElasticsearchNodeClient client = new ElasticsearchNodeClient(mock(ClusterService.class),
nodeClient);

ElasticsearchRequest request = new ElasticsearchRequest("test");
client.cleanup(request);
verify(nodeClient, never()).prepareClearScroll();
}

private ElasticsearchNodeClient mockClient(String indexName, String mappings) {
ClusterService clusterService = mockClusterService(indexName, mappings);
return new ElasticsearchNodeClient(clusterService, nodeClient);
Loading