diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java index d58b2da8908f..3cbd38480506 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveLogicalPlanner.java @@ -69,6 +69,7 @@ import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY; import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_METADATA_QUERIES; import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_DEREFERENCE_ENABLED; +import static com.facebook.presto.SystemSessionProperties.QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED; import static com.facebook.presto.common.function.OperatorType.EQUAL; import static com.facebook.presto.common.predicate.Domain.create; import static com.facebook.presto.common.predicate.Domain.multipleValues; @@ -1433,6 +1434,40 @@ public void testMaterializedViewOptimizationWithDerivedFieldsWithAlias() } } + // Currently the base to materialized view mapping is hardcoded for this test case to work + // TODO: The mapping should be fetched from metastore https://github.com/prestodb/presto/issues/16438 + @Test(enabled = false) + public void testBaseToViewConversionWithDerivedFields() + { + Session queryOptimizationWithMaterializedView = Session.builder(getSession()) + .setSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, "true") + .build(); + QueryRunner queryRunner = getQueryRunner(); + String baseTable = "lineitem_partitioned_derived_fields"; + String view = "lineitem_partitioned_view_derived_fields"; + try { + queryRunner.execute(format("CREATE TABLE %s WITH (partitioned_by = ARRAY['ds', 'shipmode']) AS " + + "SELECT discount, extendedprice, '2020-01-01' as ds, shipmode FROM lineitem WHERE orderkey < 1000 " + + "UNION ALL " + + "SELECT discount, extendedprice, '2020-01-02' as ds, shipmode FROM lineitem WHERE orderkey > 1000", baseTable)); + assertUpdate(format("CREATE MATERIALIZED VIEW %s WITH (partitioned_by = ARRAY['mvds', 'shipmode']) AS " + + "SELECT SUM(discount * extendedprice) as _discount_multi_extendedprice_ , MAX(discount*extendedprice) as _max_discount_multi_extendedprice_ , ds as mvds, shipmode " + + "FROM %s group by ds, shipmode", + view, baseTable)); + assertTrue(getQueryRunner().tableExists(getSession(), view)); + assertUpdate(format("REFRESH MATERIALIZED VIEW %s where mvds='2020-01-01'", view), 7); + String baseQuery = format("SELECT sum(discount * extendedprice) as _discount_multi_extendedprice_ , MAX(discount*extendedprice) as _max_discount_multi_extendedprice_ , ds, shipmode as method " + + "from %s group by ds, shipmode ORDER BY ds, shipmode", baseTable); + MaterializedResult optimizedQueryResult = computeActual(queryOptimizationWithMaterializedView, baseQuery); + MaterializedResult baseQueryResult = computeActual(baseQuery); + assertEquals(optimizedQueryResult, baseQueryResult); + } + finally { + queryRunner.execute("DROP TABLE IF EXISTS " + view); + queryRunner.execute("DROP TABLE IF EXISTS " + baseTable); + } + } + //TODO: Populate columnMappings to cover all joined base tables, https://github.com/prestodb/presto/issues/16220 @Test(enabled = false) public void testMaterializedViewForJoin() diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index c6dd2e858705..5f1934285094 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -198,6 +198,7 @@ public final class SystemSessionProperties public static final String OFFSET_CLAUSE_ENABLED = "offset_clause_enabled"; public static final String VERBOSE_EXCEEDED_MEMORY_LIMIT_ERRORS_ENABLED = "verbose_exceeded_memory_limit_errors_enabled"; public static final String MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED = "materialized_view_data_consistency_enabled"; + public static final String QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED = "query_optimization_with_materialized_view_enabled"; private final List> sessionProperties; @@ -1060,7 +1061,12 @@ public SystemSessionProperties( MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED, "When enabled and reading from materialized view, partition stitching is applied to achieve data consistency", featuresConfig.isMaterializedViewDataConsistencyEnabled(), - false)); + false), + booleanProperty( + QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, + "Enable query optimization with materialized view", + featuresConfig.isQueryOptimizationWithMaterializedViewEnabled(), + true)); } public static boolean isEmptyJoinOptimization(Session session) @@ -1790,4 +1796,9 @@ public static boolean isMaterializedViewDataConsistencyEnabled(Session session) { return session.getSystemProperty(MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED, Boolean.class); } + + public static boolean isQueryOptimizationWithMaterializedViewEnabled(Session session) + { + return session.getSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, Boolean.class); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 3c551b6d491c..314c188dfb3b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -202,6 +202,8 @@ public class FeaturesConfig private boolean offsetClauseEnabled; private boolean materializedViewDataConsistencyEnabled = true; + private boolean queryOptimizationWithMaterializedViewEnabled; + public enum PartitioningPrecisionStrategy { // Let Presto decide when to repartition @@ -1771,4 +1773,17 @@ public FeaturesConfig setMaterializedViewDataConsistencyEnabled(boolean material this.materializedViewDataConsistencyEnabled = materializedViewDataConsistencyEnabled; return this; } + + public boolean isQueryOptimizationWithMaterializedViewEnabled() + { + return queryOptimizationWithMaterializedViewEnabled; + } + + @Config("query-optimization-with-materialized-view-enabled") + @ConfigDescription("Experimental: Enable query optimization using materialized view. It only supports simple query formats for now.") + public FeaturesConfig setQueryOptimizationWithMaterializedViewEnabled(boolean value) + { + this.queryOptimizationWithMaterializedViewEnabled = value; + return this; + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewrite.java b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewrite.java new file mode 100644 index 000000000000..5f5422d35071 --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewrite.java @@ -0,0 +1,83 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.rewrite; + +import com.facebook.presto.Session; +import com.facebook.presto.SystemSessionProperties; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.security.AccessControl; +import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.sql.analyzer.QueryExplainer; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.AstVisitor; +import com.facebook.presto.sql.tree.Expression; +import com.facebook.presto.sql.tree.Node; +import com.facebook.presto.sql.tree.Query; +import com.facebook.presto.sql.tree.Statement; + +import java.util.List; +import java.util.Optional; + +import static com.facebook.presto.sql.rewrite.MaterializedViewOptimizationRewriteUtils.optimizeQueryUsingMaterializedView; +import static java.util.Objects.requireNonNull; + +public class MaterializedViewOptimizationRewrite + implements StatementRewrite.Rewrite +{ + @Override + public Statement rewrite( + Session session, + Metadata metadata, + SqlParser parser, + Optional queryExplainer, + Statement node, + List parameters, + AccessControl accessControl, + WarningCollector warningCollector) + { + return (Statement) new MaterializedViewOptimizationRewrite.Visitor(metadata, session, parser).process(node, null); + } + + private static final class Visitor + extends AstVisitor + { + private final Metadata metadata; + private final Session session; + private final SqlParser sqlParser; + + public Visitor( + Metadata metadata, + Session session, + SqlParser parser) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + this.session = requireNonNull(session, "session is null"); + this.sqlParser = requireNonNull(parser, "queryPreparer is null"); + } + + @Override + protected Node visitNode(Node node, Void context) + { + return node; + } + + protected Node visitQuery(Query query, Void context) + { + if (SystemSessionProperties.isQueryOptimizationWithMaterializedViewEnabled(session)) { + return optimizeQueryUsingMaterializedView(metadata, session, sqlParser, query).orElse(query); + } + return query; + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewriteUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewriteUtils.java new file mode 100644 index 000000000000..bf743223bdab --- /dev/null +++ b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/MaterializedViewOptimizationRewriteUtils.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.sql.rewrite; + +import com.facebook.presto.Session; +import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.metadata.Metadata; +import com.facebook.presto.spi.ConnectorMaterializedViewDefinition; +import com.facebook.presto.sql.analyzer.MaterializedViewCandidateExtractor; +import com.facebook.presto.sql.analyzer.MaterializedViewQueryOptimizer; +import com.facebook.presto.sql.parser.SqlParser; +import com.facebook.presto.sql.tree.QualifiedName; +import com.facebook.presto.sql.tree.Query; +import com.facebook.presto.sql.tree.Table; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class MaterializedViewOptimizationRewriteUtils +{ + private MaterializedViewOptimizationRewriteUtils() {} + + public static Optional optimizeQueryUsingMaterializedView( + Metadata metadata, + Session session, + SqlParser sqlParser, + Query node) + { + Map> baseTableToMaterializedViews = getBaseTableToMaterializedViews(); + MaterializedViewCandidateExtractor materializedViewCandidateExtractor = new MaterializedViewCandidateExtractor(session, baseTableToMaterializedViews); + materializedViewCandidateExtractor.process(node); + Set materializedViewCandidates = materializedViewCandidateExtractor.getMaterializedViewCandidates(); + if (materializedViewCandidates.isEmpty()) { + return Optional.empty(); + } + // TODO: Select the most compatible and efficient materialized view for query rewrite optimization https://github.com/prestodb/presto/issues/16431 + Query optimizedQuery = getQueryWithMaterializedViewOptimization(metadata, session, sqlParser, node, materializedViewCandidates.iterator().next()); + return Optional.of(optimizedQuery); + } + + private static Query getQueryWithMaterializedViewOptimization( + Metadata metadata, + Session session, + SqlParser sqlParser, + Query statement, + QualifiedObjectName materializedViewQualifiedObjectName) + { + ConnectorMaterializedViewDefinition materializedView = metadata.getMaterializedView(session, materializedViewQualifiedObjectName).get(); + Table materializedViewTable = new Table(QualifiedName.of(materializedView.getTable())); + + Query materializedViewDefinition = (Query) sqlParser.createStatement(materializedView.getOriginalSql()); + return (Query) new MaterializedViewQueryOptimizer(materializedViewTable, materializedViewDefinition).rewrite(statement); + } + + // TODO: The mapping should be fetched from metastore https://github.com/prestodb/presto/issues/16430 + private static Map> getBaseTableToMaterializedViews() + { + return ImmutableMap.of(); + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/StatementRewrite.java b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/StatementRewrite.java index f961692d9051..603d0067bc81 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/rewrite/StatementRewrite.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/rewrite/StatementRewrite.java @@ -35,7 +35,8 @@ public final class StatementRewrite new DescribeOutputRewrite(), new ShowQueriesRewrite(), new ShowStatsRewrite(), - new ExplainRewrite()); + new ExplainRewrite(), + new MaterializedViewOptimizationRewrite()); private StatementRewrite() {} diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 2199da00fd0b..6cd349dc1f66 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -173,7 +173,8 @@ public void testDefaults() .setPartialResultsCompletionRatioThreshold(0.5) .setOffsetClauseEnabled(false) .setPartialResultsMaxExecutionTimeMultiplier(2.0) - .setMaterializedViewDataConsistencyEnabled(true)); + .setMaterializedViewDataConsistencyEnabled(true) + .setQueryOptimizationWithMaterializedViewEnabled(false)); } @Test @@ -299,6 +300,7 @@ public void testExplicitPropertyMappings() .put("partial-results-max-execution-time-multiplier", "1.5") .put("offset-clause-enabled", "true") .put("materialized-view-data-consistency-enabled", "false") + .put("query-optimization-with-materialized-view-enabled", "true") .build(); FeaturesConfig expected = new FeaturesConfig() @@ -421,7 +423,8 @@ public void testExplicitPropertyMappings() .setPartialResultsCompletionRatioThreshold(0.9) .setOffsetClauseEnabled(true) .setPartialResultsMaxExecutionTimeMultiplier(1.5) - .setMaterializedViewDataConsistencyEnabled(false); + .setMaterializedViewDataConsistencyEnabled(false) + .setQueryOptimizationWithMaterializedViewEnabled(true); assertFullMapping(properties, expected); }