Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Merge develop to master #553

Merged
merged 49 commits into from
Jul 9, 2020
Merged
Changes from 1 commit
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
9f0d57a
[PPL] init basic codebase structure (#409)
penghuo Apr 6, 2020
13f8b85
[PPL] add basic parser support (#415)
penghuo Apr 8, 2020
fb4ab0b
Create ANTLR grammar files for PPL - Phase0 (#421)
chloe-zh Apr 9, 2020
aabb053
add the support of expression value (#432)
penghuo Apr 20, 2020
54049bd
Add support of Resolved Expression (#436)
penghuo Apr 23, 2020
ded0421
[PPL] Create ANTLR parser and implement basic syntax - Phase0 (#429)
chloe-zh Apr 23, 2020
d4147a9
change the package from plans to ast (#438)
penghuo Apr 24, 2020
242ca1f
[PPL] Implemeted full functionality of commands in phase0 (#440)
chloe-zh May 7, 2020
e7e2ecb
Feature/analyzer (#464)
penghuo May 11, 2020
ce2c08b
Implement get index mapping in ES storage engine (#469)
dai-chen May 18, 2020
30703ea
Add where stats rename command (#466)
penghuo May 18, 2020
aa890bc
[PPL] Added arithmetic binary operations in ppl parser (#474)
chloe-zh May 20, 2020
5ce4b56
[PPL] Add simple json formatter (#476)
dai-chen May 20, 2020
8e4453f
[PPL] Add support for fields command (#472)
penghuo May 20, 2020
d265e19
[PPL] Add Elasticsearch index scan operator (#475)
dai-chen May 22, 2020
9498e57
[PPL] Added PPL integration test base (#479)
chloe-zh May 22, 2020
8274edd
[PPL] Support eval command (#477)
penghuo May 22, 2020
3c3cb85
[PPL] Add Elasticsearch executor and integrate all components (#482)
dai-chen May 22, 2020
3aa0fec
[PPL] Support sort command (#483)
penghuo May 22, 2020
83e9b75
[PPL] Support dedup command (#485)
penghuo May 26, 2020
9ac0819
[PPL] Add the analyzer constructor (#487)
penghuo May 26, 2020
3a740ad
[PPL] Add ES rest client to support standalone mode (#484)
dai-chen May 26, 2020
4d226ff
Revert "[PPL] Add ES rest client to support standalone mode (#484)" (…
dai-chen May 26, 2020
5db0412
[PPL] Add ES rest client to support standalone mode (#491)
dai-chen May 26, 2020
5ae2203
Fix json formatter for consistency with SQL (#492)
dai-chen May 27, 2020
1c16f2f
[PPL] Support Count aggregator and OR operator (#493)
penghuo May 28, 2020
0f3f3f8
[PPL] Added integration tests for query analysis and search, where, f…
chloe-zh May 29, 2020
d9c05c5
[Bug] Support duplicated event for sort command (#499)
penghuo Jun 1, 2020
d81029d
[PPL] Add Doctest module (#497)
zhongnansu Jun 1, 2020
386da03
Added integration tests for dedup, rename, sort, stats commands (#498)
chloe-zh Jun 1, 2020
a261188
Merge branch 'master' into feature/ppl-merge-master
dai-chen Jun 1, 2020
d5f0e96
Enfore google java style (#502)
penghuo Jun 3, 2020
64e8562
Move old SQL code to legacy module (#506)
dai-chen Jun 4, 2020
3186242
[Bug Fix] Remove the GET request description in the SQL doc (#511)
penghuo Jun 11, 2020
c8fdbeb
Enable to rename multiple fields in syntax (#514)
chloe-zh Jun 11, 2020
7d53d82
[PPL] Add PPL documentation. (#516)
penghuo Jun 12, 2020
2a217ed
[PPL] Added syntax check exception respect to syntax analysis errors …
chloe-zh Jun 18, 2020
88a9c5d
[PPL] Support expression in syntax (#524)
chloe-zh Jun 22, 2020
dc02438
Bug fix, support long type for aggregation (#529)
penghuo Jun 22, 2020
d42dcbd
Move the index mapping definitions to standalone json files (#526)
pakio Jun 23, 2020
0fad590
[PPL] Support expression and boolean operators (#518)
chloe-zh Jun 26, 2020
087da88
Add new sql module (#527)
dai-chen Jun 26, 2020
df16ffe
[PPL] Add Resource monitor to avoid OOM (#533)
penghuo Jun 29, 2020
d60997b
Enforce comparison test in new SQL module (#536)
dai-chen Jul 1, 2020
05d3699
Support LIKE operator (#534)
chloe-zh Jul 8, 2020
dddef2e
Add select expression without query support (#542)
dai-chen Jul 8, 2020
bd379cf
Add plugin setting for new SQL module (#544)
dai-chen Jul 8, 2020
8df9a88
Redefine the Expression interface and Type (#550)
penghuo Jul 8, 2020
785f623
Bump develop to 1.9, fix all build errors (#552)
joshuali925 Jul 9, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[PPL] Support dedup command (#485)
* dedup operator

* update deduper

* add dedup ast node

* add logical dedup

* add dedup operator to ElasticsearchIndex

* address comments
penghuo authored May 26, 2020

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
commit 83e9b75e949057347c373dfe733e6889666672d5
Original file line number Diff line number Diff line change
@@ -18,9 +18,11 @@
import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.In;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.Let;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Dedupe;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Eval;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project;
@@ -36,6 +38,7 @@
import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression;
import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator;
import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation;
import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalDedupe;
import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalEval;
import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter;
import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan;
@@ -189,4 +192,24 @@ public LogicalPlan visitSort(Sort node, AnalysisContext context) {

return new LogicalSort(child, count, sortList);
}

/** Build {@link LogicalDedupe} */
@Override
public LogicalPlan visitDedupe(Dedupe node, AnalysisContext context) {
LogicalPlan child = node.getChild().get(0).accept(this, context);
List<Argument> options = node.getOptions();
// Todo, refactor the option.
Integer allowedDuplication = (Integer) options.get(0).getValue().getValue();
Boolean keepEmpty = (Boolean) options.get(1).getValue().getValue();
Boolean consecutive = (Boolean) options.get(2).getValue().getValue();

return new LogicalDedupe(
child,
node.getFields().stream()
.map(f -> expressionAnalyzer.analyze(f, context))
.collect(Collectors.toList()),
allowedDuplication,
keepEmpty,
consecutive);
}
}
Original file line number Diff line number Diff line change
@@ -32,6 +32,7 @@
import com.amazon.opendistroforelasticsearch.sql.ast.expression.QualifiedName;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedAttribute;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Dedupe;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Eval;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project;
@@ -155,4 +156,8 @@ public T visitLet(Let node, C context) {
public T visitSort(Sort node, C context) {
return visitChildren(node, context);
}

public T visitDedupe(Dedupe node, C context) {
return visitChildren(node, context);
}
}
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@
import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedAttribute;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Dedupe;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Eval;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter;
import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project;
@@ -215,7 +216,6 @@ public static List<Argument> defaultStatsArgs() {
public static List<Argument> defaultDedupArgs() {
return exprList(
argument("number", intLiteral(1)),
argument("keepevents", booleanLiteral(false)),
argument("keepempty", booleanLiteral(false)),
argument("consecutive", booleanLiteral(false)));
}
@@ -235,4 +235,8 @@ public static List<Argument> defaultSortFieldArgs() {
public static Sort sort(UnresolvedPlan input, List<Argument> options, Field... sorts) {
return new Sort(input, options, Arrays.asList(sorts));
}

public static Dedupe dedupe(UnresolvedPlan input, List<Argument> options, Field... fields) {
return new Dedupe(input, options, Arrays.asList(fields));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.sql.ast.tree;

import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument;
import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field;
import com.google.common.collect.ImmutableList;
import java.util.List;
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import lombok.ToString;

/**
* AST node represent Dedupe operation.
*/
@Getter
@Setter
@ToString
@EqualsAndHashCode(callSuper = false)
@RequiredArgsConstructor
@AllArgsConstructor
public class Dedupe extends UnresolvedPlan {
private UnresolvedPlan child;
private final List<Argument> options;
private final List<Field> fields;

@Override
public Dedupe attach(UnresolvedPlan child) {
this.child = child;
return this;
}

@Override
public List<UnresolvedPlan> getChild() {
return ImmutableList.of(this.child);
}

@Override
public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) {
return nodeVisitor.visitDedupe(this, context);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package com.amazon.opendistroforelasticsearch.sql.planner.logical;

import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
import java.util.Arrays;
import java.util.List;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.ToString;

/**
* Logical Dedupe Plan
*/
@Getter
@ToString
@EqualsAndHashCode(callSuper = false)
@RequiredArgsConstructor
public class LogicalDedupe extends LogicalPlan {
private final LogicalPlan child;
private final List<Expression> dedupeList;
private final Integer allowedDuplication;
private final Boolean keepEmpty;
private final Boolean consecutive;

@Override
public List<LogicalPlan> getChild() {
return Arrays.asList(child);
}

@Override
public <R, C> R accept(LogicalPlanNodeVisitor<R, C> visitor, C context) {
return visitor.visitDedupe(this, context);
}
}
Original file line number Diff line number Diff line change
@@ -56,12 +56,27 @@ public static LogicalPlan remove(LogicalPlan input, ReferenceExpression... field
return new LogicalRemove(input, ImmutableSet.copyOf(fields));
}

public static LogicalPlan eval(LogicalPlan input, Pair<ReferenceExpression, Expression>... expressions) {
public static LogicalPlan eval(
LogicalPlan input, Pair<ReferenceExpression, Expression>... expressions) {
return new LogicalEval(input, Arrays.asList(expressions));
}

public static LogicalPlan sort(LogicalPlan input,
Integer count, Pair<SortOption, Expression>... sorts) {
public static LogicalPlan sort(
LogicalPlan input, Integer count, Pair<SortOption, Expression>... sorts) {
return new LogicalSort(input, count, Arrays.asList(sorts));
}

public static LogicalPlan dedupe(LogicalPlan input, Expression... fields) {
return dedupe(input, 1, false, false, fields);
}

public static LogicalPlan dedupe(
LogicalPlan input,
int allowedDuplication,
boolean keepEmpty,
boolean consecutive,
Expression... fields) {
return new LogicalDedupe(
input, Arrays.asList(fields), allowedDuplication, keepEmpty, consecutive);
}
}
Original file line number Diff line number Diff line change
@@ -38,6 +38,10 @@ public R visitAggregation(LogicalAggregation plan, C context) {
return visitNode(plan, context);
}

public R visitDedupe(LogicalDedupe plan, C context) {
return visitNode(plan, context);
}

public R visitRename(LogicalRename plan, C context) {
return visitNode(plan, context);
}
Loading