Skip to content

Commit

Permalink
Add query optimization using materialized view
Browse files Browse the repository at this point in the history
Adding query optimization using materialized view. It consumes existing
candidate extractor and query optimizations utilities to optimize the
original queries if possible. This optimization is controlled by a hidden
configuration, which is disabled by default.
  • Loading branch information
Julian Zhuoran Zhao authored and highker committed Jul 20, 2021
1 parent 84e58ca commit 43b96f0
Show file tree
Hide file tree
Showing 7 changed files with 226 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
import static com.facebook.presto.SystemSessionProperties.JOIN_REORDERING_STRATEGY;
import static com.facebook.presto.SystemSessionProperties.OPTIMIZE_METADATA_QUERIES;
import static com.facebook.presto.SystemSessionProperties.PUSHDOWN_DEREFERENCE_ENABLED;
import static com.facebook.presto.SystemSessionProperties.QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED;
import static com.facebook.presto.common.function.OperatorType.EQUAL;
import static com.facebook.presto.common.predicate.Domain.create;
import static com.facebook.presto.common.predicate.Domain.multipleValues;
Expand Down Expand Up @@ -1433,6 +1434,40 @@ public void testMaterializedViewOptimizationWithDerivedFieldsWithAlias()
}
}

// Currently the base to materialized view mapping is hardcoded for this test case to work
// TODO: The mapping should be fetched from metastore https://github.com/prestodb/presto/issues/16438
@Test(enabled = false)
public void testBaseToViewConversionWithDerivedFields()
{
Session queryOptimizationWithMaterializedView = Session.builder(getSession())
.setSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, "true")
.build();
QueryRunner queryRunner = getQueryRunner();
String baseTable = "lineitem_partitioned_derived_fields";
String view = "lineitem_partitioned_view_derived_fields";
try {
queryRunner.execute(format("CREATE TABLE %s WITH (partitioned_by = ARRAY['ds', 'shipmode']) AS " +
"SELECT discount, extendedprice, '2020-01-01' as ds, shipmode FROM lineitem WHERE orderkey < 1000 " +
"UNION ALL " +
"SELECT discount, extendedprice, '2020-01-02' as ds, shipmode FROM lineitem WHERE orderkey > 1000", baseTable));
assertUpdate(format("CREATE MATERIALIZED VIEW %s WITH (partitioned_by = ARRAY['mvds', 'shipmode']) AS " +
"SELECT SUM(discount * extendedprice) as _discount_multi_extendedprice_ , MAX(discount*extendedprice) as _max_discount_multi_extendedprice_ , ds as mvds, shipmode " +
"FROM %s group by ds, shipmode",
view, baseTable));
assertTrue(getQueryRunner().tableExists(getSession(), view));
assertUpdate(format("REFRESH MATERIALIZED VIEW %s where mvds='2020-01-01'", view), 7);
String baseQuery = format("SELECT sum(discount * extendedprice) as _discount_multi_extendedprice_ , MAX(discount*extendedprice) as _max_discount_multi_extendedprice_ , ds, shipmode as method " +
"from %s group by ds, shipmode ORDER BY ds, shipmode", baseTable);
MaterializedResult optimizedQueryResult = computeActual(queryOptimizationWithMaterializedView, baseQuery);
MaterializedResult baseQueryResult = computeActual(baseQuery);
assertEquals(optimizedQueryResult, baseQueryResult);
}
finally {
queryRunner.execute("DROP TABLE IF EXISTS " + view);
queryRunner.execute("DROP TABLE IF EXISTS " + baseTable);
}
}

//TODO: Populate columnMappings to cover all joined base tables, https://github.com/prestodb/presto/issues/16220
@Test(enabled = false)
public void testMaterializedViewForJoin()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ public final class SystemSessionProperties
public static final String OFFSET_CLAUSE_ENABLED = "offset_clause_enabled";
public static final String VERBOSE_EXCEEDED_MEMORY_LIMIT_ERRORS_ENABLED = "verbose_exceeded_memory_limit_errors_enabled";
public static final String MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED = "materialized_view_data_consistency_enabled";
public static final String QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED = "query_optimization_with_materialized_view_enabled";

private final List<PropertyMetadata<?>> sessionProperties;

Expand Down Expand Up @@ -1060,7 +1061,12 @@ public SystemSessionProperties(
MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED,
"When enabled and reading from materialized view, partition stitching is applied to achieve data consistency",
featuresConfig.isMaterializedViewDataConsistencyEnabled(),
false));
false),
booleanProperty(
QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED,
"Enable query optimization with materialized view",
featuresConfig.isQueryOptimizationWithMaterializedViewEnabled(),
true));
}

public static boolean isEmptyJoinOptimization(Session session)
Expand Down Expand Up @@ -1790,4 +1796,9 @@ public static boolean isMaterializedViewDataConsistencyEnabled(Session session)
{
return session.getSystemProperty(MATERIALIZED_VIEW_DATA_CONSISTENCY_ENABLED, Boolean.class);
}

public static boolean isQueryOptimizationWithMaterializedViewEnabled(Session session)
{
return session.getSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, Boolean.class);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,8 @@ public class FeaturesConfig
private boolean offsetClauseEnabled;
private boolean materializedViewDataConsistencyEnabled = true;

private boolean queryOptimizationWithMaterializedViewEnabled;

public enum PartitioningPrecisionStrategy
{
// Let Presto decide when to repartition
Expand Down Expand Up @@ -1771,4 +1773,17 @@ public FeaturesConfig setMaterializedViewDataConsistencyEnabled(boolean material
this.materializedViewDataConsistencyEnabled = materializedViewDataConsistencyEnabled;
return this;
}

public boolean isQueryOptimizationWithMaterializedViewEnabled()
{
return queryOptimizationWithMaterializedViewEnabled;
}

@Config("query-optimization-with-materialized-view-enabled")
@ConfigDescription("Experimental: Enable query optimization using materialized view. It only supports simple query formats for now.")
public FeaturesConfig setQueryOptimizationWithMaterializedViewEnabled(boolean value)
{
this.queryOptimizationWithMaterializedViewEnabled = value;
return this;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.sql.rewrite;

import com.facebook.presto.Session;
import com.facebook.presto.SystemSessionProperties;
import com.facebook.presto.metadata.Metadata;
import com.facebook.presto.security.AccessControl;
import com.facebook.presto.spi.WarningCollector;
import com.facebook.presto.sql.analyzer.QueryExplainer;
import com.facebook.presto.sql.parser.SqlParser;
import com.facebook.presto.sql.tree.AstVisitor;
import com.facebook.presto.sql.tree.Expression;
import com.facebook.presto.sql.tree.Node;
import com.facebook.presto.sql.tree.Query;
import com.facebook.presto.sql.tree.Statement;

import java.util.List;
import java.util.Optional;

import static com.facebook.presto.sql.rewrite.MaterializedViewOptimizationRewriteUtils.optimizeQueryUsingMaterializedView;
import static java.util.Objects.requireNonNull;

public class MaterializedViewOptimizationRewrite
implements StatementRewrite.Rewrite
{
@Override
public Statement rewrite(
Session session,
Metadata metadata,
SqlParser parser,
Optional<QueryExplainer> queryExplainer,
Statement node,
List<Expression> parameters,
AccessControl accessControl,
WarningCollector warningCollector)
{
return (Statement) new MaterializedViewOptimizationRewrite.Visitor(metadata, session, parser).process(node, null);
}

private static final class Visitor
extends AstVisitor<Node, Void>
{
private final Metadata metadata;
private final Session session;
private final SqlParser sqlParser;

public Visitor(
Metadata metadata,
Session session,
SqlParser parser)
{
this.metadata = requireNonNull(metadata, "metadata is null");
this.session = requireNonNull(session, "session is null");
this.sqlParser = requireNonNull(parser, "queryPreparer is null");
}

@Override
protected Node visitNode(Node node, Void context)
{
return node;
}

protected Node visitQuery(Query query, Void context)
{
if (SystemSessionProperties.isQueryOptimizationWithMaterializedViewEnabled(session)) {
return optimizeQueryUsingMaterializedView(metadata, session, sqlParser, query).orElse(query);
}
return query;
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.sql.rewrite;

import com.facebook.presto.Session;
import com.facebook.presto.common.QualifiedObjectName;
import com.facebook.presto.metadata.Metadata;
import com.facebook.presto.spi.ConnectorMaterializedViewDefinition;
import com.facebook.presto.sql.analyzer.MaterializedViewCandidateExtractor;
import com.facebook.presto.sql.analyzer.MaterializedViewQueryOptimizer;
import com.facebook.presto.sql.parser.SqlParser;
import com.facebook.presto.sql.tree.QualifiedName;
import com.facebook.presto.sql.tree.Query;
import com.facebook.presto.sql.tree.Table;
import com.google.common.collect.ImmutableMap;

import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

public class MaterializedViewOptimizationRewriteUtils
{
private MaterializedViewOptimizationRewriteUtils() {}

public static Optional<Query> optimizeQueryUsingMaterializedView(
Metadata metadata,
Session session,
SqlParser sqlParser,
Query node)
{
Map<QualifiedObjectName, List<QualifiedObjectName>> baseTableToMaterializedViews = getBaseTableToMaterializedViews();
MaterializedViewCandidateExtractor materializedViewCandidateExtractor = new MaterializedViewCandidateExtractor(session, baseTableToMaterializedViews);
materializedViewCandidateExtractor.process(node);
Set<QualifiedObjectName> materializedViewCandidates = materializedViewCandidateExtractor.getMaterializedViewCandidates();
if (materializedViewCandidates.isEmpty()) {
return Optional.empty();
}
// TODO: Select the most compatible and efficient materialized view for query rewrite optimization https://github.com/prestodb/presto/issues/16431
Query optimizedQuery = getQueryWithMaterializedViewOptimization(metadata, session, sqlParser, node, materializedViewCandidates.iterator().next());
return Optional.of(optimizedQuery);
}

private static Query getQueryWithMaterializedViewOptimization(
Metadata metadata,
Session session,
SqlParser sqlParser,
Query statement,
QualifiedObjectName materializedViewQualifiedObjectName)
{
ConnectorMaterializedViewDefinition materializedView = metadata.getMaterializedView(session, materializedViewQualifiedObjectName).get();
Table materializedViewTable = new Table(QualifiedName.of(materializedView.getTable()));

Query materializedViewDefinition = (Query) sqlParser.createStatement(materializedView.getOriginalSql());
return (Query) new MaterializedViewQueryOptimizer(materializedViewTable, materializedViewDefinition).rewrite(statement);
}

// TODO: The mapping should be fetched from metastore https://github.com/prestodb/presto/issues/16430
private static Map<QualifiedObjectName, List<QualifiedObjectName>> getBaseTableToMaterializedViews()
{
return ImmutableMap.of();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ public final class StatementRewrite
new DescribeOutputRewrite(),
new ShowQueriesRewrite(),
new ShowStatsRewrite(),
new ExplainRewrite());
new ExplainRewrite(),
new MaterializedViewOptimizationRewrite());

private StatementRewrite() {}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,8 @@ public void testDefaults()
.setPartialResultsCompletionRatioThreshold(0.5)
.setOffsetClauseEnabled(false)
.setPartialResultsMaxExecutionTimeMultiplier(2.0)
.setMaterializedViewDataConsistencyEnabled(true));
.setMaterializedViewDataConsistencyEnabled(true)
.setQueryOptimizationWithMaterializedViewEnabled(false));
}

@Test
Expand Down Expand Up @@ -299,6 +300,7 @@ public void testExplicitPropertyMappings()
.put("partial-results-max-execution-time-multiplier", "1.5")
.put("offset-clause-enabled", "true")
.put("materialized-view-data-consistency-enabled", "false")
.put("query-optimization-with-materialized-view-enabled", "true")
.build();

FeaturesConfig expected = new FeaturesConfig()
Expand Down Expand Up @@ -421,7 +423,8 @@ public void testExplicitPropertyMappings()
.setPartialResultsCompletionRatioThreshold(0.9)
.setOffsetClauseEnabled(true)
.setPartialResultsMaxExecutionTimeMultiplier(1.5)
.setMaterializedViewDataConsistencyEnabled(false);
.setMaterializedViewDataConsistencyEnabled(false)
.setQueryOptimizationWithMaterializedViewEnabled(true);
assertFullMapping(properties, expected);
}

Expand Down

0 comments on commit 43b96f0

Please sign in to comment.