diff --git a/build.gradle b/build.gradle index d1c253c1dd..846bff7d34 100644 --- a/build.gradle +++ b/build.gradle @@ -8,8 +8,6 @@ buildscript { ext { opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT") spring_version = "5.3.22" - jackson_version = "2.14.1" - jackson_databind_version = "2.14.1" isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') @@ -69,6 +67,10 @@ plugins { id 'jacoco' } +// import versions defined in https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchJavaPlugin.java#L94 +// versions https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/version.properties +apply plugin: 'opensearch.java' + // Repository on root level is for dependencies that project code depends on. And this block must be placed after plugins{} repositories { mavenLocal() diff --git a/core/build.gradle b/core/build.gradle index fe7126fed5..0b8ffc422c 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -40,9 +40,9 @@ dependencies { api group: 'org.apache.commons', name: 'commons-lang3', version: '3.10' api group: 'com.facebook.presto', name: 'presto-matching', version: '0.240' api group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1' - api "com.fasterxml.jackson.core:jackson-core:${jackson_version}" - api "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" - api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}" + api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api project(':common') testImplementation('org.junit.jupiter:junit-jupiter:5.6.2') diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index fc425c6c20..a95932f35a 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -334,6 +334,10 @@ public static FunctionExpression dayofyear(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.DAYOFYEAR, expressions); } + public static FunctionExpression day_of_month(Expression... expressions) { + return compile(FunctionProperties.None, BuiltinFunctionName.DAY_OF_MONTH, expressions); + } + public static FunctionExpression day_of_year(Expression... expressions) { return compile(FunctionProperties.None, BuiltinFunctionName.DAY_OF_YEAR, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index a111f672af..26d7b2880e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -57,7 +57,6 @@ import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.ExpressionEvaluationException; -import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.BuiltinFunctionRepository; import org.opensearch.sql.expression.function.DefaultFunctionResolver; @@ -100,7 +99,8 @@ public void register(BuiltinFunctionRepository repository) { repository.register(date_sub()); repository.register(day()); repository.register(dayName()); - repository.register(dayOfMonth()); + repository.register(dayOfMonth(BuiltinFunctionName.DAYOFMONTH)); + repository.register(dayOfMonth(BuiltinFunctionName.DAY_OF_MONTH)); repository.register(dayOfWeek()); repository.register(dayOfYear(BuiltinFunctionName.DAYOFYEAR)); repository.register(dayOfYear(BuiltinFunctionName.DAY_OF_YEAR)); @@ -336,8 +336,8 @@ private DefaultFunctionResolver dayName() { /** * DAYOFMONTH(STRING/DATE/DATETIME/TIMESTAMP). return the day of the month (1-31). */ - private DefaultFunctionResolver dayOfMonth() { - return define(BuiltinFunctionName.DAYOFMONTH.getName(), + private DefaultFunctionResolver dayOfMonth(BuiltinFunctionName name) { + return define(name.getName(), impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, DATE), impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, DATETIME), impl(nullMissingHandling(DateTimeFunction::exprDayOfMonth), INTEGER, TIMESTAMP), diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index b23c7613d6..1435e8a556 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -67,6 +67,7 @@ public enum BuiltinFunctionName { DAY(FunctionName.of("day")), DAYNAME(FunctionName.of("dayname")), DAYOFMONTH(FunctionName.of("dayofmonth")), + DAY_OF_MONTH(FunctionName.of("day_of_month")), DAYOFWEEK(FunctionName.of("dayofweek")), DAYOFYEAR(FunctionName.of("dayofyear")), DAY_OF_YEAR(FunctionName.of("day_of_year")), diff --git a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java index 4a5276418d..4a6d4d8222 100644 --- a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java @@ -35,6 +35,7 @@ import org.opensearch.sql.planner.physical.ValuesOperator; import org.opensearch.sql.planner.physical.WindowOperator; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; /** * Default implementor for implementing logical to physical translation. "Default" here means all @@ -129,6 +130,11 @@ public PhysicalPlan visitTableScanBuilder(TableScanBuilder plan, C context) { return plan.build(); } + @Override + public PhysicalPlan visitTableWriteBuilder(TableWriteBuilder plan, C context) { + return plan.build(visitChild(plan, context)); + } + @Override public PhysicalPlan visitRelation(LogicalRelation node, C context) { throw new UnsupportedOperationException("Storage engine is responsible for " diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index b18e099afa..a192966287 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -32,6 +32,10 @@ @UtilityClass public class LogicalPlanDSL { + public static LogicalPlan write(LogicalPlan input, Table table, List columns) { + return new LogicalWrite(input, table, columns); + } + public static LogicalPlan aggregation( LogicalPlan input, List aggregatorList, List groupByList) { return new LogicalAggregation(input, aggregatorList, groupByList); diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java index 0386eb6e2a..9a41072fe7 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -7,6 +7,7 @@ package org.opensearch.sql.planner.logical; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; /** * The visitor of {@link LogicalPlan}. @@ -28,6 +29,14 @@ public R visitTableScanBuilder(TableScanBuilder plan, C context) { return visitNode(plan, context); } + public R visitWrite(LogicalWrite plan, C context) { + return visitNode(plan, context); + } + + public R visitTableWriteBuilder(TableWriteBuilder plan, C context) { + return visitNode(plan, context); + } + public R visitFilter(LogicalFilter plan, C context) { return visitNode(plan, context); } diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalWrite.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalWrite.java new file mode 100644 index 0000000000..496e6009e3 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalWrite.java @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.logical; + +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.storage.Table; + +/** + * Logical operator for insert statement. + */ +@EqualsAndHashCode(callSuper = true) +@Getter +@ToString +public class LogicalWrite extends LogicalPlan { + + /** Table that handles the write operation. */ + private final Table table; + + /** Optional column name list specified in insert statement. */ + private final List columns; + + /** + * Construct a logical write with given child node, table and column name list. + */ + public LogicalWrite(LogicalPlan child, Table table, List columns) { + super(Collections.singletonList(child)); + this.table = table; + this.columns = columns; + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitWrite(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java index f241e76993..70847b869b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizer.java @@ -17,6 +17,7 @@ import org.opensearch.sql.planner.optimizer.rule.PushFilterUnderSort; import org.opensearch.sql.planner.optimizer.rule.read.CreateTableScanBuilder; import org.opensearch.sql.planner.optimizer.rule.read.TableScanPushDown; +import org.opensearch.sql.planner.optimizer.rule.write.CreateTableWriteBuilder; /** * {@link LogicalPlan} Optimizer. @@ -55,7 +56,8 @@ public static LogicalPlanOptimizer create() { TableScanPushDown.PUSH_DOWN_SORT, TableScanPushDown.PUSH_DOWN_LIMIT, TableScanPushDown.PUSH_DOWN_HIGHLIGHT, - TableScanPushDown.PUSH_DOWN_PROJECT)); + TableScanPushDown.PUSH_DOWN_PROJECT, + new CreateTableWriteBuilder())); } /** diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java index 0ba478594a..856d8df7ea 100644 --- a/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/pattern/Patterns.java @@ -20,6 +20,7 @@ import org.opensearch.sql.planner.logical.LogicalProject; import org.opensearch.sql.planner.logical.LogicalRelation; import org.opensearch.sql.planner.logical.LogicalSort; +import org.opensearch.sql.planner.logical.LogicalWrite; import org.opensearch.sql.storage.Table; import org.opensearch.sql.storage.read.TableScanBuilder; @@ -110,4 +111,14 @@ public static Property table() { ? Optional.of(((LogicalRelation) plan).getTable()) : Optional.empty()); } + + /** + * Logical write with table field. + */ + public static Property writeTable() { + return Property.optionalProperty("table", + plan -> plan instanceof LogicalWrite + ? Optional.of(((LogicalWrite) plan).getTable()) + : Optional.empty()); + } } diff --git a/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/write/CreateTableWriteBuilder.java b/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/write/CreateTableWriteBuilder.java new file mode 100644 index 0000000000..4fbf676862 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/planner/optimizer/rule/write/CreateTableWriteBuilder.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.planner.optimizer.rule.write; + +import static org.opensearch.sql.planner.optimizer.pattern.Patterns.writeTable; + +import com.facebook.presto.matching.Capture; +import com.facebook.presto.matching.Captures; +import com.facebook.presto.matching.Pattern; +import lombok.Getter; +import lombok.experimental.Accessors; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalWrite; +import org.opensearch.sql.planner.optimizer.Rule; +import org.opensearch.sql.storage.Table; +import org.opensearch.sql.storage.write.TableWriteBuilder; + +/** + * Rule that replaces logical write operator with {@link TableWriteBuilder} for later optimization + * and transforming to physical operator. + */ +public class CreateTableWriteBuilder implements Rule { + + /** Capture the table inside matched logical relation operator. */ + private final Capture capture; + + /** Pattern that matches logical relation operator. */ + @Accessors(fluent = true) + @Getter + private final Pattern pattern; + + /** + * Construct create table write builder rule. + */ + public CreateTableWriteBuilder() { + this.capture = Capture.newCapture(); + this.pattern = Pattern.typeOf(LogicalWrite.class) + .with(writeTable().capturedAs(capture)); + } + + @Override + public LogicalPlan apply(LogicalWrite plan, Captures captures) { + return captures.get(capture).createWriteBuilder(plan); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 63dd05cc6b..d4bc4a1ea9 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -7,6 +7,7 @@ package org.opensearch.sql.planner.physical; import org.opensearch.sql.storage.TableScanOperator; +import org.opensearch.sql.storage.write.TableWriteOperator; /** * The visitor of {@link PhysicalPlan}. @@ -36,6 +37,10 @@ public R visitTableScan(TableScanOperator node, C context) { return visitNode(node, context); } + public R visitTableWrite(TableWriteOperator node, C context) { + return visitNode(node, context); + } + public R visitProject(ProjectOperator node, C context) { return visitNode(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/storage/Table.java b/core/src/main/java/org/opensearch/sql/storage/Table.java index ae0aaaf17b..496281fa8d 100644 --- a/core/src/main/java/org/opensearch/sql/storage/Table.java +++ b/core/src/main/java/org/opensearch/sql/storage/Table.java @@ -10,8 +10,10 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.executor.streaming.StreamingSource; import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalWrite; import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; /** * Table. @@ -73,6 +75,17 @@ default TableScanBuilder createScanBuilder() { return null; // TODO: Enforce all subclasses to implement this later } + /* + * Create table write builder for logical to physical transformation. + * + * @param plan logical write plan + * @return table write builder + */ + default TableWriteBuilder createWriteBuilder(LogicalWrite plan) { + throw new UnsupportedOperationException( + "Write operation is not supported on current table"); + } + /** * Translate {@link Table} to {@link StreamingSource} if possible. */ diff --git a/core/src/main/java/org/opensearch/sql/storage/write/TableWriteBuilder.java b/core/src/main/java/org/opensearch/sql/storage/write/TableWriteBuilder.java new file mode 100644 index 0000000000..54dfa5d557 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/storage/write/TableWriteBuilder.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.storage.write; + +import java.util.Collections; +import org.opensearch.sql.planner.logical.LogicalPlan; +import org.opensearch.sql.planner.logical.LogicalPlanNodeVisitor; +import org.opensearch.sql.planner.physical.PhysicalPlan; + +/** + * A {@link TableWriteBuilder} represents transition state between logical planning and physical + * planning for table write operator. The concrete implementation class gets involved in the logical + * optimization through this abstraction and thus transform to specific {@link TableWriteOperator} + * in a certain storage engine. + */ +public abstract class TableWriteBuilder extends LogicalPlan { + + /** + * Construct table write builder with child node. + */ + public TableWriteBuilder(LogicalPlan child) { + super(Collections.singletonList(child)); + } + + /** + * Build table write operator with given child node. + * + * @param child child operator node + * @return table write operator + */ + public abstract TableWriteOperator build(PhysicalPlan child); + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitTableWriteBuilder(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/storage/write/TableWriteOperator.java b/core/src/main/java/org/opensearch/sql/storage/write/TableWriteOperator.java new file mode 100644 index 0000000000..92cdc6eb41 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/storage/write/TableWriteOperator.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.storage.write; + +import java.util.Collections; +import java.util.List; +import lombok.RequiredArgsConstructor; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlanNodeVisitor; + +/** + * {@link TableWriteOperator} is the abstraction for data source to implement different physical + * write operator on a data source. This is also to avoid "polluting" physical plan visitor by + * concrete table scan implementation. + */ +@RequiredArgsConstructor +public abstract class TableWriteOperator extends PhysicalPlan { + + /** Input physical node. */ + protected final PhysicalPlan input; + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitTableWrite(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + /** + * Explain the execution plan. + * + * @return explain output + */ + public abstract String explain(); +} diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java index 092b64d5d7..20119797ad 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/DateTimeFunctionTest.java @@ -430,6 +430,64 @@ public void dayOfMonth() { assertEquals(integerValue(8), eval(expression)); } + public void testDayOfMonthWithUnderscores(FunctionExpression dateExpression, int dayOfMonth) { + assertEquals(INTEGER, dateExpression.type()); + assertEquals(integerValue(dayOfMonth), eval(dateExpression)); + } + + @Test + public void dayOfMonthWithUnderscores() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + + FunctionExpression expression1 = DSL.dayofmonth(DSL.literal(new ExprDateValue("2020-08-07"))); + FunctionExpression expression2 = DSL.dayofmonth(DSL.literal("2020-07-08")); + + assertAll( + () -> testDayOfMonthWithUnderscores(expression1, 7), + () -> assertEquals("dayofmonth(DATE '2020-08-07')", expression1.toString()), + + () -> testDayOfMonthWithUnderscores(expression2, 8), + () -> assertEquals("dayofmonth(\"2020-07-08\")", expression2.toString()) + + ); + } + + public void testInvalidDayOfMonth(String date) { + FunctionExpression expression = DSL.day_of_month(DSL.literal(new ExprDateValue(date))); + eval(expression); + } + + @Test + public void dayOfMonthWithUnderscoresLeapYear() { + lenient().when(nullRef.valueOf(env)).thenReturn(nullValue()); + lenient().when(missingRef.valueOf(env)).thenReturn(missingValue()); + + //Feb. 29 of a leap year + testDayOfMonthWithUnderscores(DSL.day_of_month(DSL.literal("2020-02-29")), 29); + + //Feb. 29 of a non-leap year + assertThrows(SemanticCheckException.class, () -> testInvalidDayOfMonth("2021-02-29")); + } + + @Test + public void dayOfMonthWithUnderscoresInvalidArguments() { + lenient().when(nullRef.type()).thenReturn(DATE); + lenient().when(missingRef.type()).thenReturn(DATE); + assertEquals(nullValue(), eval(DSL.day_of_month(nullRef))); + assertEquals(missingValue(), eval(DSL.day_of_month(missingRef))); + + //40th day of the month + assertThrows(SemanticCheckException.class, () -> testInvalidDayOfMonth("2021-02-40")); + + //13th month of the year + assertThrows(SemanticCheckException.class, () -> testInvalidDayOfMonth("2021-13-40")); + + //incorrect format + assertThrows(SemanticCheckException.class, () -> testInvalidDayOfMonth("asdfasdfasdf")); + } + @Test public void dayOfWeek() { when(nullRef.type()).thenReturn(DATE); diff --git a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java index 2322e4684e..017cfb60ea 100644 --- a/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/DefaultImplementorTest.java @@ -58,6 +58,8 @@ import org.opensearch.sql.storage.Table; import org.opensearch.sql.storage.TableScanOperator; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; +import org.opensearch.sql.storage.write.TableWriteOperator; @ExtendWith(MockitoExtension.class) class DefaultImplementorTest { @@ -212,4 +214,17 @@ public TableScanOperator build() { }; assertEquals(tableScanOperator, tableScanBuilder.accept(implementor, null)); } + + @Test + public void visitTableWriteBuilderShouldBuildTableWriteOperator() { + LogicalPlan child = values(); + TableWriteOperator tableWriteOperator = Mockito.mock(TableWriteOperator.class); + TableWriteBuilder logicalPlan = new TableWriteBuilder(child) { + @Override + public TableWriteOperator build(PhysicalPlan child) { + return tableWriteOperator; + } + }; + assertEquals(tableWriteOperator, logicalPlan.accept(implementor, null)); + } } diff --git a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java index 33c6b02c38..341bcbc29e 100644 --- a/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -12,6 +12,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -31,9 +32,12 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.Aggregator; import org.opensearch.sql.expression.window.WindowDefinition; +import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.Table; import org.opensearch.sql.storage.TableScanOperator; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; +import org.opensearch.sql.storage.write.TableWriteOperator; /** * Todo. Temporary added for UT coverage, Will be removed. @@ -83,6 +87,19 @@ public TableScanOperator build() { assertNull(tableScanBuilder.accept(new LogicalPlanNodeVisitor() { }, null)); + LogicalPlan write = LogicalPlanDSL.write(null, table, Collections.emptyList()); + assertNull(write.accept(new LogicalPlanNodeVisitor() { + }, null)); + + TableWriteBuilder tableWriteBuilder = new TableWriteBuilder(null) { + @Override + public TableWriteOperator build(PhysicalPlan child) { + return null; + } + }; + assertNull(tableWriteBuilder.accept(new LogicalPlanNodeVisitor() { + }, null)); + LogicalPlan filter = LogicalPlanDSL.filter(relation, expression); assertNull(filter.accept(new LogicalPlanNodeVisitor() { }, null)); diff --git a/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java b/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java index e2510ec464..7516aa1809 100644 --- a/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/optimizer/LogicalPlanOptimizerTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.planner.optimizer; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; @@ -20,6 +21,8 @@ import static org.opensearch.sql.planner.logical.LogicalPlanDSL.project; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.relation; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.sort; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.values; +import static org.opensearch.sql.planner.logical.LogicalPlanDSL.write; import com.google.common.collect.ImmutableList; import java.util.Collections; @@ -39,6 +42,7 @@ import org.opensearch.sql.planner.physical.PhysicalPlan; import org.opensearch.sql.storage.Table; import org.opensearch.sql.storage.read.TableScanBuilder; +import org.opensearch.sql.storage.write.TableWriteBuilder; @ExtendWith(MockitoExtension.class) class LogicalPlanOptimizerTest { @@ -270,6 +274,37 @@ public PhysicalPlan implement(LogicalPlan plan) { ); } + @Test + void table_support_write_builder_should_be_replaced() { + Mockito.reset(table, tableScanBuilder); + TableWriteBuilder writeBuilder = Mockito.mock(TableWriteBuilder.class); + when(table.createWriteBuilder(any())).thenReturn(writeBuilder); + + assertEquals( + writeBuilder, + optimize(write(values(), table, Collections.emptyList())) + ); + } + + @Test + void table_not_support_write_builder_should_report_error() { + Mockito.reset(table, tableScanBuilder); + Table table = new Table() { + @Override + public Map getFieldTypes() { + return null; + } + + @Override + public PhysicalPlan implement(LogicalPlan plan) { + return null; + } + }; + + assertThrows(UnsupportedOperationException.class, + () -> table.createWriteBuilder(null)); + } + private LogicalPlan optimize(LogicalPlan plan) { final LogicalPlanOptimizer optimizer = LogicalPlanOptimizer.create(); final LogicalPlan optimize = optimizer.optimize(plan); diff --git a/core/src/test/java/org/opensearch/sql/planner/optimizer/pattern/PatternsTest.java b/core/src/test/java/org/opensearch/sql/planner/optimizer/pattern/PatternsTest.java index 61d192362a..9f90fd8d05 100644 --- a/core/src/test/java/org/opensearch/sql/planner/optimizer/pattern/PatternsTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/optimizer/pattern/PatternsTest.java @@ -35,5 +35,6 @@ void source_is_empty() { void table_is_empty() { plan = Mockito.mock(LogicalFilter.class); assertFalse(Patterns.table().getFunction().apply(plan).isPresent()); + assertFalse(Patterns.writeTable().getFunction().apply(plan).isPresent()); } } diff --git a/core/src/test/java/org/opensearch/sql/storage/write/TableWriteOperatorTest.java b/core/src/test/java/org/opensearch/sql/storage/write/TableWriteOperatorTest.java new file mode 100644 index 0000000000..8780b08276 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/storage/write/TableWriteOperatorTest.java @@ -0,0 +1,70 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.storage.write; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.Collections; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.planner.physical.PhysicalPlan; +import org.opensearch.sql.planner.physical.PhysicalPlanNodeVisitor; + +@ExtendWith(MockitoExtension.class) +class TableWriteOperatorTest { + + @Mock + private PhysicalPlan child; + + private TableWriteOperator tableWrite; + + @BeforeEach + void setUp() { + tableWrite = new TableWriteOperator(child) { + @Override + public String explain() { + return "explain"; + } + + @Override + public boolean hasNext() { + return false; + } + + @Override + public ExprValue next() { + return null; + } + }; + } + + @Test + void testAccept() { + Boolean isVisited = tableWrite.accept(new PhysicalPlanNodeVisitor<>() { + @Override + protected Boolean visitNode(PhysicalPlan node, Object context) { + return (node instanceof TableWriteOperator); + } + + @Override + public Boolean visitTableWrite(TableWriteOperator node, Object context) { + return super.visitTableWrite(node, context); + } + }, null); + + assertTrue(isVisited); + } + + @Test + void testGetChild() { + assertEquals(Collections.singletonList(child), tableWrite.getChild()); + } +} \ No newline at end of file diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 843d6c7e45..bff4975ccb 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1365,7 +1365,7 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER -Synonyms: DAYOFMONTH +Synonyms: `DAYOFMONTH`_, `DAY_OF_MONTH`_ Example:: @@ -1413,7 +1413,7 @@ Argument type: STRING/DATE/DATETIME/TIMESTAMP Return type: INTEGER -Synonyms: DAY +Synonyms: `DAY`_, `DAY_OF_MONTH`_ Example:: @@ -1425,6 +1425,54 @@ Example:: | 26 | +----------------------------------+ +DAY_OF_MONTH +---------- + +Description +>>>>>>>>>>> + +Usage: day_of_month(date) extracts the day of the month for date, in the range 1 to 31. The dates with value 0 such as '0000-00-00' or '2008-00-00' are invalid. + +Argument type: STRING/DATE/DATETIME/TIMESTAMP + +Return type: INTEGER + +Synonyms: `DAY`_, `DAYOFMONTH`_ + +Example:: + + os> SELECT DAY_OF_MONTH('2020-08-26') + fetched rows / total rows = 1/1 + +------------------------------+ + | DAY_OF_MONTH('2020-08-26') | + |------------------------------| + | 26 | + +------------------------------+ + + os> SELECT DAY_OF_MONTH(DATE('2020-08-26')) + fetched rows / total rows = 1/1 + +------------------------------------+ + | DAY_OF_MONTH(DATE('2020-08-26')) | + |------------------------------------| + | 26 | + +------------------------------------+ + + os> SELECT DAY_OF_MONTH(TIMESTAMP('2020-08-26 00:00:00')) + fetched rows / total rows = 1/1 + +--------------------------------------------------+ + | DAY_OF_MONTH(TIMESTAMP('2020-08-26 00:00:00')) | + |--------------------------------------------------| + | 26 | + +--------------------------------------------------+ + + os> SELECT DAY_OF_MONTH(DATETIME('2020-08-26 00:00:00')) + fetched rows / total rows = 1/1 + +-------------------------------------------------+ + | DAY_OF_MONTH(DATETIME('2020-08-26 00:00:00')) | + |-------------------------------------------------| + | 26 | + +-------------------------------------------------+ + DAYOFWEEK --------- diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 8adfd84b40..900bb1ae90 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -65,11 +65,11 @@ configurations.all { // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 3503877d64..fd638911ee 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -206,6 +206,48 @@ public void testDayOfMonth() throws IOException { verifyDataRows(result, rows(16)); } + @Test + public void testDayOfMonthWithUnderscores() throws IOException { + JSONObject result = executeQuery("select day_of_month(date('2020-09-16'))"); + verifySchema(result, schema("day_of_month(date('2020-09-16'))", null, "integer")); + verifyDataRows(result, rows(16)); + + result = executeQuery("select day_of_month('2020-09-16')"); + verifySchema(result, schema("day_of_month('2020-09-16')", null, "integer")); + verifyDataRows(result, rows(16)); + } + + @Test + public void testDayOfMonthAliasesReturnTheSameResults() throws IOException { + JSONObject result1 = executeQuery("SELECT dayofmonth(date('2022-11-22'))"); + JSONObject result2 = executeQuery("SELECT day_of_month(date('2022-11-22'))"); + verifyDataRows(result1, rows(22)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT dayofmonth(CAST(date0 AS date)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT day_of_month(CAST(date0 AS date)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT dayofmonth(datetime(CAST(time0 AS STRING))) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT day_of_month(datetime(CAST(time0 AS STRING))) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT dayofmonth(CAST(time0 AS STRING)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT day_of_month(CAST(time0 AS STRING)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + + result1 = executeQuery(String.format( + "SELECT dayofmonth(CAST(datetime0 AS timestamp)) FROM %s", TEST_INDEX_CALCS)); + result2 = executeQuery(String.format( + "SELECT day_of_month(CAST(datetime0 AS timestamp)) FROM %s", TEST_INDEX_CALCS)); + result1.getJSONArray("datarows").similar(result2.getJSONArray("datarows")); + } @Test public void testDayOfWeek() throws IOException { JSONObject result = executeQuery("select dayofweek(date('2020-09-16'))"); diff --git a/opensearch/build.gradle b/opensearch/build.gradle index 7ad7d63546..6eeab86fff 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -32,9 +32,9 @@ dependencies { api project(':core') api group: 'org.opensearch', name: 'opensearch', version: "${opensearch_version}" implementation "io.github.resilience4j:resilience4j-retry:1.5.0" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" implementation group: 'org.json', name: 'json', version:'20180813' compileOnly group: 'org.opensearch.client', name: 'opensearch-rest-high-level-client', version: "${opensearch_version}" implementation group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}" diff --git a/plugin/build.gradle b/plugin/build.gradle index d689da0f57..4097b2c7e5 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -85,14 +85,14 @@ configurations.all { // conflict with spring-jcl exclude group: "commons-logging", module: "commons-logging" // enforce 2.12.6, https://github.com/opensearch-project/sql/issues/424 - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${jackson_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" // enforce 1.1.3, https://www.whitesourcesoftware.com/vulnerability-database/WS-2019-0379 resolutionStrategy.force 'commons-codec:commons-codec:1.13' resolutionStrategy.force 'com.google.guava:guava:31.0.1-jre' - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${jackson_version}" - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-smile:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.6.0" resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib-common:1.6.0" resolutionStrategy.force "com.squareup.okhttp3:okhttp:4.9.3" @@ -111,9 +111,9 @@ compileTestJava { dependencies { api group: 'org.springframework', name: 'spring-beans', version: "${spring_version}" - api "com.fasterxml.jackson.core:jackson-core:${jackson_version}" - api "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" - api "com.fasterxml.jackson.core:jackson-annotations:${jackson_version}" + api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" + api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" + api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" api project(":ppl") api project(':legacy') diff --git a/prometheus/build.gradle b/prometheus/build.gradle index b11c23fd25..7cf1e56085 100644 --- a/prometheus/build.gradle +++ b/prometheus/build.gradle @@ -17,9 +17,9 @@ repositories { dependencies { api project(':core') implementation "io.github.resilience4j:resilience4j-retry:1.5.0" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" implementation group: 'com.squareup.okhttp3', name: 'okhttp', version: '4.9.3' implementation 'com.github.babbel:okhttp-aws-signer:1.0.2' implementation group: 'org.json', name: 'json', version: '20180813' diff --git a/protocol/build.gradle b/protocol/build.gradle index 9c41fbf101..5d32a235ea 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -30,9 +30,9 @@ plugins { dependencies { implementation group: 'com.google.guava', name: 'guava', version: '31.0.1-jre' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${jackson_version}" - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${jackson_databind_version}" - implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${jackson_version}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: "${versions.jackson}" + implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: "${versions.jackson_databind}" + implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-cbor', version: "${versions.jackson}" implementation 'com.google.code.gson:gson:2.8.9' implementation project(':core') implementation project(':opensearch') @@ -44,7 +44,7 @@ dependencies { } configurations.all { - resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${jackson_databind_version}" + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}" } test { diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 58d4be1813..e5b2dae97e 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -244,6 +244,7 @@ datetimeConstantLiteral : CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP + | DAY_OF_MONTH | DAY_OF_YEAR | LOCALTIME | LOCALTIMESTAMP diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index bfd0f93ec9..67a13cf6e5 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -197,6 +197,12 @@ public void can_parse_week_of_year_functions() { assertNotNull(parser.parse("SELECT week_of_year('2022-11-18')")); } + @Test + public void can_parse_dayofmonth_functions() { + assertNotNull(parser.parse("SELECT dayofmonth('2022-11-18')")); + assertNotNull(parser.parse("SELECT day_of_month('2022-11-18')")); + } + @Test public void can_parse_dayofyear_functions() { assertNotNull(parser.parse("SELECT dayofyear('2022-11-18')"));